def load(self): self._common_kwargs = self._build_common_election_kwargs() self._common_kwargs['reporting_level'] = 'precinct' # Store result instances for bulk loading results = [] with self._file_handle as csvfile: reader = unicodecsv.DictReader(csvfile) next(reader, None) for row in reader: if self._skip_row(row): continue if row['votes'] == 'X': continue rr_kwargs = self._common_kwargs.copy() rr_kwargs['primary_party'] = row['party'].strip() rr_kwargs.update(self._build_contest_kwargs(row)) rr_kwargs.update(self._build_candidate_kwargs(row)) jurisdiction = row['precinct'].strip() county_ocd_id = [c for c in self.datasource._jurisdictions() if c['county'].strip().upper() == row['county'].strip().upper()][0]['ocd_id'] rr_kwargs.update({ 'party': row['party'].strip(), 'jurisdiction': jurisdiction, 'parent_jurisdiction': row['county'], 'ocd_id': "{}/precinct:{}".format(county_ocd_id, ocd_type_id(jurisdiction)), 'office': row['office'].strip(), 'district': row['district'].strip(), 'votes': int(float(row['votes'])) }) results.append(RawResult(**rr_kwargs)) RawResult.objects.insert(results)
def write_data(self, file, outfile): """ :param file: :param outfile: :return: """ columns = self.get_column_names() # Ensure removal of output file try: os.remove(outfile) except OSError: pass csvwriter.open(outfile) csvwriter.write(columns) global_data = OrderedDict([(col, '') for col in columns]) csvgen = csvreader(file) # Skip first two lines try: headers = next(csvgen) # Fix file header headers[0] = 'V1' next(csvgen) except: pass for row in csvgen: subject_data = OrderedDict(list(zip(headers, row))) self._write_data(subject_data, global_data, columns, outfile) csvwriter.close()
def _process_callback(self, callback, args): """Processes a callback to a normal or a streaming function. In constrast with :meth:`sijax.response.BaseResponse._process_callback` which only processes normal requests properly, this can process both normal and streaming functions. Normal functions are the typical Sijax response functions, which don't flush content to the browser, but only push commands to the buffer list. Those commands are to be flushed by Sijax when the response function exits. Streaming functions are the typical Comet response functions, which are generators (they use yield to flush content). Basically this function can be seen as a converter from either a generator (streaming function) or a string (normal function) to a generator. """ response = self._perform_handler_call(callback, args) if isinstance(response, GeneratorType): # Real streaming function using a generator to flush while True: # we don't really care what it yields.. next(response) if len(self._commands) != 0: yield self._flush() else: # Normal (non-streaming) function # Let's flush implicitly for such functions if len(self._commands) != 0: yield self._flush()
def parse(token, next): s = token[2] if len(s) < 2 or not (s[0] == s[-1] == '"'): token[-1].error('No end quote on string', token) s = quoted_splitter(s[1:-1]) result = parse_quoted_str(token, s[0]) if len(s) > 1: result = [result] append = result.append s = iter(s) next = s.__next__ next() for special in s: nonmatch = next() remap = quoted_mapper(special) if remap is None: if len(special) == 6: uni = int(special[2:], 16) if 0xd800 <= uni <= 0xdbff and allow_double: uni, nonmatch = parse_double_unicode(uni, nonmatch, next, token) remap = parse_encoded_chr(uni) else: return badstring(token, special) append(remap) append(parse_quoted_str(token, nonmatch)) result = parse_join_str(result) if cachestrings: result = token[-1].stringcache(result, result) return result
def loadMovie(self, filename, log=None): """Load a movie from file :Parameters: filename: string The name of the file, including path if necessary Brings up a warning if avbin is not found on the computer. After the file is loaded MovieStim.duration is updated with the movie duration (in seconds). """ try: self._movie = pyglet.media.load(filename, streaming=True) except Exception as e: # pyglet.media.riff is N/A if avbin is available, and then # actual exception would get masked with a new one for unknown # (sub)module riff, thus catching any exception and tuning msg # up if it has to do anything with avbin estr = str(e) msg = '' if "avbin" in estr.lower(): msg = ("\nIt seems that avbin was not installed correctly." "\nPlease fetch/install it from " "http://code.google.com/p/avbin/.") emsg = "Caught exception '%s' while loading file '%s'.%s" raise IOError(emsg % (estr, filename, msg)) self._player.queue(self._movie) self.duration = self._movie.duration while self._player.source != self._movie: next(self._player) self.status = NOT_STARTED self._player.pause() # start 'playing' on the next draw command self.filename = filename logAttrib(self, log, 'movie', filename)
def execute(self, context): self.hive = HiveCliHook(hive_cli_conn_id=self.hive_cli_conn_id) self.s3 = S3Hook(s3_conn_id=self.s3_conn_id) logging.info("Downloading S3 file") if self.wildcard_match: if not self.s3.check_for_wildcard_key(self.s3_key): raise AirflowException("No key matches {0}".format(self.s3_key)) s3_key_object = self.s3.get_wildcard_key(self.s3_key) else: if not self.s3.check_for_key(self.s3_key): raise AirflowException( "The key {0} does not exists".format(self.s3_key)) s3_key_object = self.s3.get_key(self.s3_key) with NamedTemporaryFile("w") as f: logging.info("Dumping S3 key {0} contents to local" " file {1}".format(s3_key_object.key, f.name)) s3_key_object.get_contents_to_file(f) f.flush() self.s3.connection.close() if not self.headers: logging.info("Loading file into Hive") self.hive.load_file( f.name, self.hive_table, field_dict=self.field_dict, create=self.create, partition=self.partition, delimiter=self.delimiter, recreate=self.recreate) else: with open(f.name, 'r') as tmpf: if self.check_headers: header_l = tmpf.readline() header_line = header_l.rstrip() header_list = header_line.split(self.delimiter) field_names = list(self.field_dict.keys()) test_field_match = [h1.lower() == h2.lower() for h1, h2 in zip(header_list, field_names)] if not all(test_field_match): logging.warning("Headers do not match field names" "File headers:\n {header_list}\n" "Field names: \n {field_names}\n" "".format(**locals())) raise AirflowException("Headers do not match the " "field_dict keys") with NamedTemporaryFile("w") as f_no_headers: tmpf.seek(0) next(tmpf) for line in tmpf: f_no_headers.write(line) f_no_headers.flush() logging.info("Loading file without headers into Hive") self.hive.load_file( f_no_headers.name, self.hive_table, field_dict=self.field_dict, create=self.create, partition=self.partition, delimiter=self.delimiter, recreate=self.recreate)
def read_json_dict(firsttok, next): result = new_object() append = result.append while 1: token = next() t0 = token[1] if t0 == '}': if result and disallow_trailing_commas: error('Unexpected trailing comma', token) break key = json_value_dispatch(t0, bad_dict_key)(token, next) if disallow_nonstring_keys and not isinstance(key, basestring): error('Non-string key %s not supported' % repr(key), token) token = next() t0 = token[1] if t0 != ':': error('Expected ":" after dict key %s' % repr(key), token) token = next() t0 = token[1] value = json_value_dispatch(t0, bad_dict_value)(token, next) append([key, value]) delim = next() t0 = delim[1] if t0 == ',': continue if t0 != '}': if t0 == '@': error('Unterminated dict (no matching "}")', firsttok) error('Expected "," or "}"', delim) break return result.get_result(firsttok)
def testOgrDriverList(self): # test with drivers in recommended order drivers = QgsVectorFileWriter.ogrDriverList(QgsVectorFileWriter.SortRecommended) self.assertEqual(drivers[0].longName, 'GeoPackage') self.assertEqual(drivers[0].driverName, 'GPKG') self.assertEqual(drivers[1].longName, 'ESRI Shapefile') self.assertEqual(drivers[1].driverName, 'ESRI Shapefile') self.assertTrue('ODS' in [f.driverName for f in drivers]) # ensure that XLSX comes before SQLite, because we should sort on longName, not driverName! ms_xlsx_index = next(i for i, v in enumerate(drivers) if v.driverName == 'XLSX') sqlite_index = next(i for i, v in enumerate(drivers) if v.driverName == 'SQLite') self.assertLess(ms_xlsx_index, sqlite_index) self.assertIn('[XLSX]', drivers[ms_xlsx_index].longName) # alphabetical sorting drivers2 = QgsVectorFileWriter.ogrDriverList(QgsVectorFileWriter.VectorFormatOptions()) self.assertTrue(drivers2[0].longName < drivers2[1].longName) self.assertCountEqual([d.driverName for d in drivers], [d.driverName for d in drivers2]) self.assertNotEqual(drivers2[0].driverName, 'GPKG') # skip non-spatial formats = QgsVectorFileWriter.ogrDriverList(QgsVectorFileWriter.SkipNonSpatialFormats) self.assertFalse('ODS' in [f.driverName for f in formats])
def _visit(self, node): iterator = self._get_iterator(node) n = next(iterator) while n: result = n.accept(self) n = next(iterator) return result
def __init__(self, lexer, parser, parent): Token.__init__(self, self.__class__.__name__, lexer, parser, parent) # Create a grammar depending on the delimiting character. tok_type, delimiter = lexer.token() escaped_delimiter = '\\' + delimiter data = r'[^\r\n\\' + escaped_delimiter + ']+' delimiter_re = re.compile(escaped_delimiter) data_re = re.compile(data) grammar_with_delim = grammar_c[:] grammar_with_delim.append(('string_data', data_re)) grammar_with_delim.append(('string_delimiter', delimiter_re)) lexer.set_grammar(grammar_with_delim) # Begin parsing the string. lexer.expect(self, 'string_delimiter') self.string = '' while 1: if lexer.current_is('string_data'): self.string += lexer.token()[1] next(lexer) elif lexer.current_is('escaped_data'): self.string += self._escape(lexer.token()[1]) next(lexer) elif lexer.next_if('string_delimiter'): break else: ttype = lexer.token()[0] lexer.syntax_error('Expected string but got %s' % ttype, self) # Make sure that any variables specified in the command are declared. string_re.sub(self.variable_test_cb, self.string) self.mark_end() lexer.restore_grammar()
def exhaust_generator(gen): self.assertTrue(isinstance(gen, GeneratorType)) try: while True: next(gen) except StopIteration: pass
def getTrianglesCoordinates(self): """ A method to retrieve triplet of coordinates representing the triangles in lon,lat,height. """ triangles = [] self._computeVerticesCoordinates() indices = iter(self.indices) for i in xrange(0, len(self.indices) - 1, 3): vi1 = next(indices) vi2 = next(indices) vi3 = next(indices) triangle = ( (self._longs[vi1], self._lats[vi1], self._heights[vi1]), (self._longs[vi2], self._lats[vi2], self._heights[vi2]), (self._longs[vi3], self._lats[vi3], self._heights[vi3]) ) triangles.append(triangle) if len(list(indices)) > 0: raise Exception('Corrupted tile') return triangles
def combineCSVs(dataFile, featureFileList): import csv returnDictionary = {} with open(dataFile, "r") as dataListFile: dataListReader = csv.reader(dataListFile, delimiter=",", skipinitialspace=True) dataListHeader = next(dataListReader) for session in dataListReader: sessionWithHeader = list(zip(dataListHeader, session)) sessionDict = {} for (name, value) in sessionWithHeader: sessionDict[name] = value.strip() returnDictionary[sessionDict['sessionID']] = sessionDict returnDictionary[sessionDict['sessionID']]['featureImageDict'] = {} # initialize import ast for ft in list(featureFileList.keys()): with open(featureFileList[ft], "r") as featureListFile: featureListReader = csv.reader(featureListFile, delimiter=",", skipinitialspace=True) featureListHeader = next(featureListReader) for row in featureListReader: rowWithHeader = list(zip(featureListHeader, row)) rowFeatureDict = {} for (name, value) in rowWithHeader: rowFeatureDict[name] = value.strip() currSessionDict = returnDictionary[rowFeatureDict['sessionID']] print(rowFeatureDict) currSessionDict['featureImageDict'][ft] = ast.literal_eval(rowFeatureDict['featureImage'])[ft] returnDictionary[rowFeatureDict['sessionID']] = currSessionDict return returnDictionary
def __load_multipolygon(tokens, string): """ Has similar inputs and return value to to :func:`__load_point`, except is for handling MULTIPOLYGON geometry. :returns: A GeoJSON `dict` MultiPolygon representation of the WKT ``string``. """ open_paren = next(tokens) if not open_paren == '(': raise ValueError(INVALID_WKT_FMT % string) polygons = [] while True: try: poly = __load_polygon(tokens, string) polygons.append(poly['coordinates']) t = next(tokens) if t == ')': # we're done; no more polygons. break except StopIteration: # If we reach this, the WKT is not valid. raise ValueError(INVALID_WKT_FMT % string) return dict(type='MultiPolygon', coordinates=polygons)
def ranges(self, start, end): while start >= self.end: self.start, self.end, self.value = next(self) yield start, min(self.end, end), self.value while end > self.end: self.start, self.end, self.value = next(self) yield self.start, min(self.end, end), self.value
def __iter__(self): res = super(Cursor, self).__iter__() while True: if self.callback: yield self.callback(next(res)) else: yield next(res)
def __load_multilinestring(tokens, string): """ Has similar inputs and return value to to :func:`__load_point`, except is for handling MULTILINESTRING geometry. :returns: A GeoJSON `dict` MultiLineString representation of the WKT ``string``. """ open_paren = next(tokens) if not open_paren == '(': raise ValueError(INVALID_WKT_FMT % string) linestrs = [] while True: try: linestr = __load_linestring(tokens, string) linestrs.append(linestr['coordinates']) t = next(tokens) if t == ')': # we're done; no more linestrings. break except StopIteration: # If we reach this, the WKT is not valid. raise ValueError(INVALID_WKT_FMT % string) return dict(type='MultiLineString', coordinates=linestrs)
def test_dimension_operators(self): result = un.dimensionless # Arbitrary starting expression dim_iter = cycle(dimensions) val_iter = cycle(single_values) for op in self.ops: if op is pow: val = int(next(val_iter) * 10) # Scale the value close to 10 to avoid overflow errors if val != 0.0: val = val / 10 ** round(np.log10(abs(val))) dim = np_dim = int(val) else: dim = next(dim_iter) np_dim = 10 ** self._np_array(dim) np_result = np.log10(op(10 ** self._np_array(result), np_dim)) new_result = op(result, dim) op_str = ("{}({}, {})".format(op.__name__, result, dim)) self.assertIsInstance(result, un.Dimension, op_str + " did not return a Dimension") self.assertTrue( all(self._np_array(new_result) == np_result), "{} not equal between Dimension ({}) and numpy ({})" .format(op_str, self._np_array(new_result), np_result)) result = new_result
def collision(self, ball): """ Determines if the ball hits this obstacle :param ball: An instance of :class:`BallModel` :type ball: :class:`BallModel` """ self._double_collision = False if ball.position[0] - ball.radius > self.max_x: return False if ball.position[0] + ball.radius < self.min_x: return False if ball.position[1] - ball.radius > self.max_y: return False if ball.position[1] + ball.radius < self.min_y: return False a, b = tee(np.vstack([np.array(self.points), self.points[0]])) next(b, None) intercept_found = False for pt_pair in zip(a, b): if self._intercept_edge(pt_pair, ball): if intercept_found: # Ball has hit a corner self._intercept = self._select_edge( pt_pair, self._intercept, ball) self._double_collision = True else: self._intercept = pt_pair intercept_found = True return intercept_found
def write_seg(dframe, sample_id=None, chrom_ids=None): """Format a dataframe or list of dataframes as SEG. To put multiple samples into one SEG table, pass `dframe` and `sample_id` as equal-length lists of data tables and sample IDs in matching order. """ assert sample_id is not None if isinstance(dframe, pd.DataFrame): first = dframe first_sid = sample_id sids = dframes = None else: assert not isinstance(sample_id, basestring) dframes = iter(dframe) sids = iter(sample_id) first = next(dframes) first_sid = next(sids) if chrom_ids in (None, True): chrom_ids = create_chrom_ids(first) results = [format_seg(first, first_sid, chrom_ids)] if dframes is not None: # Unpack matching lists of data and sample IDs results.extend( format_seg(subframe, sid, chrom_ids) for subframe, sid in zip_longest(dframes, sids)) return pd.concat(results)
def test_unit_unit_operators(self): result = un.unitless # Arbitrary starting expression unit_iter = cycle(units) val_iter = cycle(single_values) for op in self.ops: if op is pow: val = int(next(val_iter) * 10) # Scale the value close to 10 to avoid overflow errors if val != 0: val = val / 10 ** round(np.log10(abs(val))) unit = dim = power = val else: unit = next(unit_iter) dim = unit.dimension power = 10 ** unit.power dim_result = op(result.dimension, dim) new_result = op(result, unit) power_result = np.log10(op(float(10 ** result.power), float(power))) op_str = ("{}({}, {})".format(op.__name__, result, unit)) self.assertIsInstance(result, un.Unit, op_str + " did not return a Dimension") self.assertEqual( new_result.dimension, dim_result, "Dimension of {} not equal between Unit ({}) and explicit " "({})".format(op_str, new_result.dimension, dim_result)) self.assertEqual( new_result.power, power_result, "Power of {} not equal between Unit ({}) and explicit ({})" .format(op_str, new_result.power, power_result)) result = new_result
def decode_item(next, token): if token == b"i": # integer: "i" value "e" data = int(next()) if next() != b"e": raise ValueError elif token == b"s": # string: "s" value (virtual tokens) data = next() # Strings in torrent file are defined as utf-8 encoded try: data = data.decode('utf-8') except UnicodeDecodeError as e: # The pieces field is a byte string, and should be left as such. pass elif token == b"l" or token == b"d": # container: "l" (or "d") values "e" data = [] tok = next() while tok != b"e": data.append(decode_item(next, tok)) tok = next() if token == b"d": data = dict(list(zip(data[0::2], data[1::2]))) else: raise ValueError return data
def parse_xml_file(xml_file_path): try: root = ET.parse(xml_file_path).getroot() for testcase in root.iter('testcase'): test_info = {} try: test_info.update({'time': float(testcase.attrib.get('time'))}) except (TypeError, ValueError): test_info.update({'time': None}) for attribute in testcase_attributes: test_info[attribute] = testcase.attrib.get(attribute) result = SUCCESS if next(testcase.iter('error'), None) is not None: result = ERROR elif next(testcase.iter('failure'), None) is not None: result = FAILURE elif next(testcase.iter('skipped'), None) is not None: result = SKIPPED test_info.update({'result_code': result}) tests_in_path.update({testcase.attrib.get('name', ''): test_info}) except (ET.ParseError, ValueError) as e: error_handler(ParseError(xml_file_path, e))
def _get_javac_args_from_zinc_args(zinc_args): javac_args = [] i = iter(zinc_args) arg = next(i, None) output_dir = None while arg is not None: arg = arg.strip() if arg in ['-d', '-cp', '-classpath']: # These are passed through from zinc to javac. javac_args.append(arg) javac_args.append(next(i)) if arg == '-d': output_dir = javac_args[-1] elif arg.startswith('-C'): javac_args.append(arg[2:]) elif arg.endswith('.java'): javac_args.append(arg) arg = next(i, None) # Strip output dir from classpaths. If we don't then javac will read annotation definitions # from there instead of from the source files, which will cause the vnames to reflect the .class # file instead of the .java file. if output_dir: for i, a in enumerate(javac_args): if a in ['-cp', '-classpath']: javac_args[i + 1] = ':'.join([p for p in javac_args[i + 1].split(':') if p != output_dir]) return javac_args
def test_entities_iterators(self, _): # Given intent_entities = { "entity1": { "utterances": { "entity 1": "entity 1", "entity 11": "entity 11", "entity 111": "entity 111", } }, "entity2": { "utterances": { "entity 2": "entity 2", "entity 22": "entity 22", "entity 222": "entity 222", } } } random_state = np.random.RandomState(1) # Then it_dict = get_entities_iterators(intent_entities, random_state) # When self.assertIn("entity1", it_dict) expected_seq = ["entity 1", "entity 11", "entity 111"] seq = [next(it_dict["entity1"]) for _ in range(len(expected_seq))] self.assertListEqual(expected_seq, sorted(seq)) self.assertIn("entity2", it_dict) expected_seq = ["entity 2", "entity 22", "entity 222"] seq = [next(it_dict["entity2"]) for _ in range(len(expected_seq))] self.assertListEqual(expected_seq, sorted(seq))
def pairwise(iterable): """s -> (s0,s1), (s1,s2), (s2,s3), ... copied from documentation """ a, b = tee(iterable) next(b, None) return zip(a, b)
def testStart(self): self.testStop() self.assertEqual(len(self.pipeline), 1) item1 = object() self.pipeline.appendleft(item1) self.assertEqual(next(self.pipeline), None) self.pipeline.start() self.assertEqual(next(self.pipeline), item1)
def testNestedInsert(self): tg = QgsTransactionGroup() tg.addLayer(self.vl) self.vl.startEditing() it = self.vl.getFeatures() f = next(it) f['pk'] = NULL self.vl.addFeature(f) # Should not deadlock during an active iteration f = next(it)
def parse_fn_name(t): if next(t) == '(': # fn pointer name assert next(t) == '*' name=next(t) assert next(t) == ')' else: name=t.current() return name
def test_open_zip_returns_realpath_on_badzipfile(self): # In case of file corruption, deleting a Pants-constructed symlink would not resolve the error. with temporary_file() as not_zip: with temporary_dir() as tempdir: file_symlink = os.path.join(tempdir, 'foo') os.symlink(not_zip.name, file_symlink) self.assertEquals(os.path.realpath(file_symlink), os.path.realpath(not_zip.name)) with self.assertRaisesRegexp(zipfile.BadZipfile, r'{}'.format(not_zip.name)): next(open_zip(file_symlink).gen)
def pop(self, last=True): if not self: raise KeyError('set is empty') key = next(reversed(self)) if last else next(iter(self)) self.discard(key) return key
def range_pair(field, cat, fq_filter, iterable, end, collection_facet): # e.g. counts":["0",17430,"1000",1949,"2000",671,"3000",404,"4000",243,"5000",165],"gap":1000,"start":0,"end":6000} pairs = [] selected_values = [f['value'] for f in fq_filter] is_single_unit_gap = re.match( '^[\+\-]?1[A-Za-z]*$', str(collection_facet['properties']['gap'])) is not None is_up = collection_facet['properties']['sort'] == 'asc' if collection_facet['properties']['sort'] == 'asc' and ( collection_facet['type'] == 'range-up' or collection_facet['properties'].get('type') == 'range-up'): prev = None n = [] for e in iterable: if prev is not None: n.append(e) n.append(prev) prev = None else: prev = e iterable = n iterable.reverse() a, to = itertools.tee(iterable) next(to, None) counts = iterable[1::2] total_counts = counts.pop( 0) if collection_facet['properties']['sort'] == 'asc' else 0 isDate = collection_facet['properties']['isDate'] for element in a: next(to, None) to_value = next(to, end) count = next(a) if collection_facet['properties']['sort'] == 'asc': from_value = to_value to_value = element else: from_value = element pairs.append({ 'field': field, 'from': from_value if isDate else int(element), 'value': count, 'to': to_value if isDate else int(to_value), 'selected': element in selected_values, 'exclude': all([f['exclude'] for f in fq_filter if f['value'] == element]), 'is_single_unit_gap': is_single_unit_gap, 'total_counts': total_counts, 'is_up': is_up }) total_counts += counts.pop(0) if counts else 0 if collection_facet['properties']['sort'] == 'asc' and collection_facet[ 'type'] != 'range-up' and collection_facet['properties'].get( 'type') != 'range-up': pairs.reverse() return pairs
def has_with_pattern(self, re_pattern): """Return whether if there is an item with the given pattern.""" return bool(next(self.get_by_pattern(re_pattern), None))
def __next__(self): return _mongo_item_to_task(next(self.cursor))
def main(): """ Main function to handle vcenter vm os and the mapping to a policy group """ # Handling arguments args = get_args() clusters = [] if args.clusters: clusters = args.clusters debug = args.debug log_file = None if args.logfile: log_file = args.logfile mapping_file = args.mapping_file nuage_enterprise = args.nuage_enterprise nuage_host = args.nuage_host nuage_port = args.nuage_port nuage_password = None if args.nuage_password: nuage_password = args.nuage_password nuage_username = args.nuage_username remove_policygroups = args.remove_policygroups nosslcheck = args.nosslcheck verbose = args.verbose vcenter_host = args.vcenter_host vcenter_https_port = args.vcenter_https_port vcenter_password = None if args.vcenter_password: vcenter_password = args.vcenter_password vcenter_username = args.vcenter_username # Logging settings if debug: log_level = logging.DEBUG elif verbose: log_level = logging.INFO else: log_level = logging.WARNING logging.basicConfig(filename=log_file, format='%(asctime)s %(levelname)s %(message)s', level=log_level) logger = logging.getLogger(__name__) # Getting user password for Nuage connection if nuage_password is None: logger.debug( 'No command line Nuage password received, requesting Nuage password from user' ) nuage_password = getpass.getpass( prompt='Enter password for Nuage host {0:s} for user {1:s}: '. format(nuage_host, nuage_username)) # Getting user password for vCenter connection if vcenter_password is None: logger.debug( 'No command line vCenter password received, requesting vCenter password from user' ) vcenter_password = getpass.getpass( prompt='Enter password for vCenter host {0:s} for user {1:s}: '. format(vcenter_host, vcenter_username)) try: vc = None nc = None # Connecting to Nuage try: logger.info( 'Connecting to Nuage server {0:s}:{1:d} with username {2:s}'. format(nuage_host, nuage_port, nuage_username)) nc = vsdk.NUVSDSession(username=nuage_username, password=nuage_password, enterprise=nuage_enterprise, api_url="https://{0:s}:{1:d}".format( nuage_host, nuage_port)) nc.start() except IOError: pass if not nc or not nc.is_current_session(): logger.error( 'Could not connect to Nuage host {0:s} with user {1:s} and specified password' .format(nuage_host, nuage_username)) return 1 # Connecting to vCenter try: logger.info( 'Connecting to vCenter server {0:s}:{1:d} with username {2:s}'. format(vcenter_host, vcenter_https_port, vcenter_username)) if nosslcheck: vc = SmartConnectNoSSL(host=vcenter_host, user=vcenter_username, pwd=vcenter_password, port=int(vcenter_https_port)) else: vc = SmartConnect(host=vcenter_host, user=vcenter_username, pwd=vcenter_password, port=int(vcenter_https_port)) except IOError: pass if not vc: logger.error( 'Could not connect to vCenter host {0:s} with user {1:s} and specified password' .format(vcenter_host, vcenter_username)) return 1 logger.debug('Registering vCenter disconnect at exit') atexit.register(Disconnect, vc) logger.info('Connected to both Nuage & vCenter servers') except vmodl.MethodFault as e: logger.critical('Caught vmodl fault: {0:s}'.format(e.msg)) return 1 # CSV Handling if not os.path.isfile(mapping_file): logger.critical( 'Mapping file {0:s} does not exist, exiting'.format(mapping_file)) return 1 mapping_list = {} # CSV fields: # vCenter VM name regex, Policy group logger.debug('Parsing mapping file {0:s}'.format(mapping_file)) with open(mapping_file, 'rb') as maplist: mapping_list_raw = csv.reader(maplist, delimiter=',', quotechar='"') for row in mapping_list_raw: logger.debug('Found CSV row: {0:s}'.format(','.join(row))) mapping_list[row[0]] = row[1] # Getting clusters in the current vCenter logger.debug( 'Gathering all Clusters from the vCenter {0:s}'.format(vcenter_host)) content = vc.content obj_view = content.viewManager.CreateContainerView( content.rootFolder, [vim.ClusterComputeResource], True) vc_cl_list = obj_view.view obj_view.Destroy() for vc_cl in vc_cl_list: if vc_cl.name not in clusters: continue # Getting VMs in the current vCenter Cluster logger.debug('Gathering all VMs from the vCenter Cluster {0:s}'.format( vc_cl.name)) obj_view = content.viewManager.CreateContainerView( vc_cl, [vim.VirtualMachine], True) vc_vm_list = obj_view.view obj_view.Destroy() for vc_vm in vc_vm_list: # Verifying if VM matches a regex in the list logger.debug('Found VM {0:s}, checking'.format(vc_vm.name)) # If the VM is a template skip it if vc_vm.config.template: logger.debug('VM {0:s} is a template, skipping'.format( vc_vm.name)) continue # Getting VM info nc_vm_properties = {} vc_vm_nuage_enterprise = next((x for x in vc_vm.config.extraConfig if x.key == 'nuage.enterprise'), None) vc_vm_nuage_domain = next((x for x in vc_vm.config.extraConfig if x.key == 'nuage.nic0.domain'), None) vc_vm_nuage_l2domain = next((x for x in vc_vm.config.extraConfig if x.key == 'nuage.nic0.l2domain'), None) vc_vm_nuage_zone = next((x for x in vc_vm.config.extraConfig if x.key == 'nuage.nic0.zone'), None) vc_vm_nuage_network = next((x for x in vc_vm.config.extraConfig if x.key == 'nuage.nic0.network'), None) # Check if all the settings for an L3 domain are present if vc_vm_nuage_enterprise is None or vc_vm_nuage_domain is None or vc_vm_nuage_zone is None or vc_vm_nuage_network is None: # Check if it is an L2 domain if vc_vm_nuage_enterprise is None or vc_vm_nuage_l2domain is None: logger.info( 'VM {0:s} has no correct Nuage metadata set, assuming it is not a VM connected through Nuage and skipping it.' .format(vc_vm.name)) continue nc_vm_properties['name'] = vc_vm.name nc_vm_properties['os'] = vc_vm.config.guestFullName nc_vm_properties['nuage.enterprise'] = vc_vm_nuage_enterprise.value # If domain is not set, it is an l2 domain if vc_vm_nuage_domain is not None: nc_vm_properties['nuage.domain'] = vc_vm_nuage_domain.value nc_vm_properties['nuage.l2domain'] = None nc_vm_domain_name = vc_vm_nuage_domain.value else: nc_vm_properties['nuage.domain'] = None nc_vm_properties['nuage.l2domain'] = vc_vm_nuage_l2domain.value nc_vm_domain_name = vc_vm_nuage_l2domain.value if vc_vm_nuage_zone is not None: nc_vm_properties['nuage.zone'] = vc_vm_nuage_zone.value else: nc_vm_properties['nuage.zone'] = None if vc_vm_nuage_network is not None: nc_vm_properties['nuage.network'] = vc_vm_nuage_network.value else: nc_vm_properties['nuage.network'] = None logger.debug( 'VM {0:s} with OS {1:s} has following Nuage settings: Enterprise {2:s}, Domain {3:s}, Zone {4:s}, Subnet {5:s}' .format(nc_vm_properties['name'], nc_vm_properties['os'], nc_vm_properties['nuage.enterprise'], nc_vm_domain_name, nc_vm_properties['nuage.zone'], nc_vm_properties['nuage.network'])) # Getting VM MAC vc_vm_nic = next( (x for x in vc_vm.config.hardware.device if isinstance(x, vim.vm.device.VirtualEthernetCard)), None) if vc_vm_nic is None: logger.error( 'VM {0:s} has no valid network interfaces, skipping it'. format(nc_vm_properties['name'])) continue nc_vm_properties['mac'] = vc_vm_nic.macAddress logger.debug('VM {0:s} has MAC {1:s}'.format( nc_vm_properties['name'], nc_vm_properties['mac'])) # Getting Nuage vport for this VM nc_vm_properties['vm_interface'] = nc.user.vm_interfaces.get_first( filter="MAC == '{0:s}'".format(nc_vm_properties['mac'])) if nc_vm_properties['vm_interface'] is None: logger.error( 'VM {0:s} with MAC address {1:s} is not known in Nuage, skipping it' .format(nc_vm_properties['name'], nc_vm_properties['mac'])) continue # Getting Nuage vport for this VM nc_vm_properties['vport'] = vsdk.NUVPort( id=nc_vm_properties['vm_interface'].vport_id) try: nc_vm_properties['vport'].fetch() except BambouHTTPError: logger.error( 'VM {0:s} with MAC address {1:s} has a vm_interface but no vport in Nuage, this should not be possible... Skipping it' .format(nc_vm_properties['name'], nc_vm_properties['mac'])) continue logger.debug( 'Found vm_interface and vport for VM {0:s} with MAC address {1:s}' .format(nc_vm_properties['name'], nc_vm_properties['mac'])) # Checking regex's on VMs nc_vm_pgs = [] for regex in list(mapping_list.keys()): logger.debug( 'Checking regex "{0:s}" on VM {1:s} with OS {2:s}'.format( regex, nc_vm_properties['name'], nc_vm_properties['os'])) pattern = re.compile(regex) if pattern.match(nc_vm_properties['os']): logger.debug( 'Found match: regex "{0:s}" and VM OS "{1:s}", adding to the task list to hand over to Nuage.' .format(regex, nc_vm_properties['os'])) nc_vm_pgs.append(mapping_list[regex]) if len(nc_vm_pgs) > 0: logger.debug( 'Handing task over to Nuage part to set {0:d} Policy Groups on VM {1:s}' .format(len(nc_vm_pgs), nc_vm_properties['name'])) update_nuage_policy_group( logger=logger, nc=nc, nc_vm_properties=nc_vm_properties, nc_vm_pgs=nc_vm_pgs, remove_policygroups=remove_policygroups) logger.info('All done!') return 0
def __next__(this): try: return next(this.generator) except StopIteration: return None
def convert_to_beam_type(typ): """Convert a given typing type to a Beam type. Args: typ (type): typing type. Returns: type: The given type converted to a Beam type as far as we can do the conversion. Raises: ~exceptions.ValueError: The type was malformed. """ if isinstance(typ, typing.TypeVar): # This is a special case, as it's not parameterized by types. # Also, identity must be preserved through conversion (i.e. the same # TypeVar instance must get converted into the same TypeVariable instance). # A global cache should be OK as the number of distinct type variables # is generally small. if id(typ) not in _type_var_cache: _type_var_cache[id(typ)] = typehints.TypeVariable(typ.__name__) return _type_var_cache[id(typ)] elif getattr(typ, '__module__', None) != 'typing': # Only tranlsate types from the typing module. return typ type_map = [ _TypeMapEntry(match=_match_same_type(typing.Any), arity=0, beam_type=typehints.Any), _TypeMapEntry(match=_match_issubclass(typing.Dict), arity=2, beam_type=typehints.Dict), _TypeMapEntry(match=_match_is_exactly_iterable, arity=1, beam_type=typehints.Iterable), _TypeMapEntry(match=_match_issubclass(typing.List), arity=1, beam_type=typehints.List), _TypeMapEntry(match=_match_issubclass(typing.Set), arity=1, beam_type=typehints.Set), # NamedTuple is a subclass of Tuple, but it needs special handling. # We just convert it to Any for now. # This MUST appear before the entry for the normal Tuple. _TypeMapEntry(match=_match_is_named_tuple, arity=0, beam_type=typehints.Any), _TypeMapEntry(match=_match_issubclass(typing.Tuple), arity=-1, beam_type=typehints.Tuple), _TypeMapEntry(match=_match_is_union, arity=-1, beam_type=typehints.Union), ] # Find the first matching entry. matched_entry = next((entry for entry in type_map if entry.match(typ)), None) if not matched_entry: # No match: return original type. return typ if matched_entry.arity == -1: arity = _len_arg(typ) else: arity = matched_entry.arity if _len_arg(typ) != arity: raise ValueError( 'expecting type %s to have arity %d, had arity %d ' 'instead' % (str(typ), arity, _len_arg(typ))) typs = [convert_to_beam_type(_get_arg(typ, i)) for i in range(arity)] if arity == 0: # Nullary types (e.g. Any) don't accept empty tuples as arguments. return matched_entry.beam_type elif arity == 1: # Unary types (e.g. Set) don't accept 1-tuples as arguments return matched_entry.beam_type[typs[0]] else: return matched_entry.beam_type[tuple(typs)]
def processAlgorithm(self, feedback): radius = self.getParameterValue(self.DISTANCE) horizontal = self.getParameterValue(self.HORIZONTAL) output = self.getOutputFromName(self.OUTPUT_LAYER) layer = dataobjects.getObjectFromUri( self.getParameterValue(self.INPUT_LAYER)) writer = output.getVectorWriter(layer.fields(), layer.wkbType(), layer.crs()) features = vector.features(layer) total = 100.0 / len(features) duplicates = dict() for current, f in enumerate(features): wkt = f.geometry().exportToWkt() if wkt not in duplicates: duplicates[wkt] = [f.id()] else: duplicates[wkt].extend([f.id()]) feedback.setProgress(int(current * total)) current = 0 total = 100.0 / len(duplicates) feedback.setProgress(0) fullPerimeter = 2 * math.pi for (geom, fids) in list(duplicates.items()): count = len(fids) if count == 1: f = next( layer.getFeatures(QgsFeatureRequest().setFilterFid( fids[0]))) writer.addFeature(f) else: angleStep = fullPerimeter / count if count == 2 and horizontal: currentAngle = math.pi / 2 else: currentAngle = 0 old_point = QgsGeometry.fromWkt(geom).asPoint() request = QgsFeatureRequest().setFilterFids(fids).setFlags( QgsFeatureRequest.NoGeometry) for f in layer.getFeatures(request): sinusCurrentAngle = math.sin(currentAngle) cosinusCurrentAngle = math.cos(currentAngle) dx = radius * sinusCurrentAngle dy = radius * cosinusCurrentAngle new_point = QgsPoint(old_point.x() + dx, old_point.y() + dy) out_feature = QgsFeature() out_feature.setGeometry(QgsGeometry.fromPoint(new_point)) out_feature.setAttributes(f.attributes()) writer.addFeature(out_feature) currentAngle += angleStep current += 1 feedback.setProgress(int(current * total)) del writer
def processAlgorithm(self, parameters, context, feedback): layerPoints = QgsProcessingUtils.mapLayerFromString( self.getParameterValue(self.POINTS), context) layerHubs = QgsProcessingUtils.mapLayerFromString( self.getParameterValue(self.HUBS), context) fieldName = self.getParameterValue(self.FIELD) units = self.UNITS[self.getParameterValue(self.UNIT)] if layerPoints.source() == layerHubs.source(): raise GeoAlgorithmExecutionException( self.tr('Same layer given for both hubs and spokes')) fields = layerPoints.fields() fields.append(QgsField('HubName', QVariant.String)) fields.append(QgsField('HubDist', QVariant.Double)) writer = self.getOutputFromName(self.OUTPUT).getVectorWriter( fields, QgsWkbTypes.LineString, layerPoints.crs(), context) index = QgsProcessingUtils.createSpatialIndex(layerHubs, context) distance = QgsDistanceArea() distance.setSourceCrs(layerPoints.crs()) distance.setEllipsoid(QgsProject.instance().ellipsoid()) # Scan source points, find nearest hub, and write to output file features = QgsProcessingUtils.getFeatures(layerPoints, context) total = 100.0 / layerPoints.featureCount() if layerPoints.featureCount( ) else 0 for current, f in enumerate(features): src = f.geometry().boundingBox().center() neighbors = index.nearestNeighbor(src, 1) ft = next( layerHubs.getFeatures(QgsFeatureRequest().setFilterFid( neighbors[0]).setSubsetOfAttributes([fieldName], layerHubs.fields()))) closest = ft.geometry().boundingBox().center() hubDist = distance.measureLine(src, closest) attributes = f.attributes() attributes.append(ft[fieldName]) if units == 'Feet': attributes.append(hubDist * 3.2808399) elif units == 'Miles': attributes.append(hubDist * 0.000621371192) elif units == 'Kilometers': attributes.append(hubDist / 1000.0) elif units != 'Meters': attributes.append( sqrt( pow(src.x() - closest.x(), 2.0) + pow(src.y() - closest.y(), 2.0))) else: attributes.append(hubDist) feat = QgsFeature() feat.setAttributes(attributes) feat.setGeometry(QgsGeometry.fromPolyline([src, closest])) writer.addFeature(feat, QgsFeatureSink.FastInsert) feedback.setProgress(int(current * total)) del writer
def testDateTimeWriteShapefile(self): """Check writing date and time fields to an ESRI shapefile.""" ml = QgsVectorLayer( ('Point?crs=epsg:4326&field=id:int&' 'field=date_f:date&field=time_f:time&field=dt_f:datetime'), 'test', 'memory') self.assertTrue(ml.isValid()) provider = ml.dataProvider() self.assertIsNotNone(provider) ft = QgsFeature() ft.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(10, 10))) ft.setAttributes([ 1, QDate(2014, 3, 5), QTime(13, 45, 22), QDateTime(QDate(2014, 3, 5), QTime(13, 45, 22)) ]) res, features = provider.addFeatures([ft]) self.assertTrue(res) self.assertTrue(features) dest_file_name = os.path.join(str(QDir.tempPath()), 'datetime.shp') crs = QgsCoordinateReferenceSystem() crs.createFromId(4326, QgsCoordinateReferenceSystem.EpsgCrsId) write_result, error_message = QgsVectorFileWriter.writeAsVectorFormat( ml, dest_file_name, 'utf-8', crs, 'ESRI Shapefile') self.assertEqual(write_result, QgsVectorFileWriter.NoError, error_message) # Open result and check created_layer = QgsVectorLayer('{}|layerid=0'.format(dest_file_name), 'test', 'ogr') fields = created_layer.dataProvider().fields() self.assertEqual( fields.at(fields.indexFromName('date_f')).type(), QVariant.Date) # shapefiles do not support time types, result should be string self.assertEqual( fields.at(fields.indexFromName('time_f')).type(), QVariant.String) # shapefiles do not support datetime types, result should be string self.assertEqual( fields.at(fields.indexFromName('dt_f')).type(), QVariant.String) f = next(created_layer.getFeatures(QgsFeatureRequest())) date_idx = created_layer.fields().lookupField('date_f') self.assertIsInstance(f.attributes()[date_idx], QDate) self.assertEqual(f.attributes()[date_idx], QDate(2014, 3, 5)) time_idx = created_layer.fields().lookupField('time_f') # shapefiles do not support time types self.assertIsInstance(f.attributes()[time_idx], str) self.assertEqual(f.attributes()[time_idx], '13:45:22') # shapefiles do not support datetime types datetime_idx = created_layer.fields().lookupField('dt_f') self.assertIsInstance(f.attributes()[datetime_idx], str) self.assertEqual( f.attributes()[datetime_idx], QDateTime(QDate(2014, 3, 5), QTime(13, 45, 22)).toString("yyyy/MM/dd hh:mm:ss.zzz"))
def executeCmd(self, cmdVar): """Execute a command (of type opscore.actor.keyvar.CmdVar). Performs the following tasks: - Sets the command ID number - Sets the start time - Puts the command on the keyword dispatcher queue - Sends the command to the server Inputs: - cmdVar: the command, of class opscore.actor.keyvar.CmdVar Note: - Always increments cmdID because every command must have a unique command ID (even commands that go to different actors); this simplifies the dispatcher code and also makes the hub's life easier (since it can report certain kinds of failures using actor=hub). """ if not self._isConnected: errReply = self.makeReply( dataStr="Failed; Actor=%r; Cmd=%r; Text=\"not connected\"" % (cmdVar.actor, cmdVar.cmdStr), ) self._replyToCmdVar(cmdVar, errReply) return while True: if cmdVar.isRefresh: cmdID = next(self.refreshCmdIDGen) else: cmdID = next(self.userCmdIDGen) if cmdID not in self.cmdDict: break self.cmdDict[cmdID] = cmdVar cmdVar._setStartInfo(self, cmdID) try: if self.includeName: # internal actor; must specify the commander if cmdVar.forUserCmd: cmdrStr = "%s.%s " % (cmdVar.forUserCmd.cmdr, self.connection.cmdr) else: cmdrStr = "%s.%s " % (self.connection.cmdr, self.connection.cmdr) else: # external actor; do not specify the commander cmdrStr = "" fullCmdStr = "%s%d %s %s" % (cmdrStr, cmdVar.cmdID, cmdVar.actor, cmdVar.cmdStr) self.connection.writeLine(fullCmdStr) # self.logMsg ( # msgStr = fullCmdStr, # actor = cmdVar.actor, # cmdID = cmdVar.cmdID, # ) # print >> sys.stderr, "executing:", fullCmdStr except Exception as e: errReply = self.makeReply( cmdID=cmdVar.cmdID, dataStr="WriteFailed; Actor=%r; Cmd=%r; Text=%r" % (cmdVar.actor, cmdVar.cmdStr, RO.StringUtil.strFromException(e)), ) self._replyToCmdVar(cmdVar, errReply)
def _frame_alignment_base(dataset, max_displacement=None, method='correlation', n_processes=1, **method_kwargs): """Estimate whole-frame displacements based on pixel correlations. Parameters ---------- max_displacement : array see estimate_displacements Returns ------- shifts : array (2, num_frames*num_cycles)-array of integers giving the estimated displacement of each frame correlations : array (num_frames*num_cycles)-array giving the correlation of each shifted frame with the reference n_processes : int, optional Number of pool processes to spawn to parallelize frame alignment. Defaults to 1. """ if n_processes < 1: raise ValueError('n_processes must be at least 1') global namespace global lock if n_processes > 1: namespace = multiprocessing.Manager().Namespace() else: namespace = Struct() namespace.offset = np.zeros(3, dtype=int) namespace.pixel_counts = np.zeros(dataset.frame_shape) # TODO: int? namespace.pixel_sums = np.zeros(dataset.frame_shape).astype('float64') # NOTE: float64 gives nan when divided by 0 namespace.shifts = [ np.zeros(seq.shape[:2] + (3, ), dtype=int) for seq in dataset ] namespace.correlations = [np.empty(seq.shape[:2]) for seq in dataset] namespace.min_shift = np.zeros(3) namespace.max_shift = np.zeros(3) lock = multiprocessing.Lock() if n_processes > 1: pool = multiprocessing.Pool(processes=n_processes, maxtasksperchild=1) for cycle_idx, cycle in zip(it.count(), dataset): #CZ added #tmp = zip(it.count(), cycle, it.repeat(cycle_idx), # it.repeat(method), it.repeat(max_displacement)) #print(list(tmp)[5]) chunksize = min(1 + old_div(len(cycle), n_processes), 200) if n_processes > 1: map_generator = pool.imap_unordered( _align_frame, zip(it.count(), cycle, it.repeat(cycle_idx), it.repeat(method), it.repeat(max_displacement), it.repeat(method_kwargs)), chunksize=chunksize) else: map_generator = map( _align_frame, zip(it.count(), cycle, it.repeat(cycle_idx), it.repeat(method), it.repeat(max_displacement), it.repeat(method_kwargs))) # Loop over generator and calculate frame alignments while True: try: next(map_generator) except StopIteration: break if n_processes > 1: pool.close() pool.join() def _align_planes(shifts): """Align planes to minimize shifts between them.""" mean_shift = nanmean(np.concatenate(shifts), axis=0) # calculate alteration of shape (num_planes, dim) alteration = (mean_shift - mean_shift[0]).astype(int) for seq in shifts: seq -= alteration shifts = [s[..., 1:] for s in namespace.shifts] correlations = namespace.correlations del namespace.pixel_counts del namespace.pixel_sums del namespace.shifts del namespace.correlations _align_planes(shifts) return shifts, correlations
def processAlgorithm(self, feedback): layer = dataobjects.getObjectFromUri( self.getParameterValue(self.VECTOR)) pointCount = float(self.getParameterValue(self.POINT_NUMBER)) minDistance = float(self.getParameterValue(self.MIN_DISTANCE)) fields = QgsFields() fields.append(QgsField('id', QVariant.Int, '', 10, 0)) writer = self.getOutputFromName(self.OUTPUT).getVectorWriter( fields, QgsWkbTypes.Point, layer.crs()) nPoints = 0 nIterations = 0 maxIterations = pointCount * 200 featureCount = layer.featureCount() total = 100.0 / pointCount index = QgsSpatialIndex() points = dict() da = QgsDistanceArea() request = QgsFeatureRequest() random.seed() while nIterations < maxIterations and nPoints < pointCount: # pick random feature fid = random.randint(0, featureCount - 1) f = next( layer.getFeatures( request.setFilterFid(fid).setSubsetOfAttributes([]))) fGeom = f.geometry() if fGeom.isMultipart(): lines = fGeom.asMultiPolyline() # pick random line lineId = random.randint(0, len(lines) - 1) vertices = lines[lineId] else: vertices = fGeom.asPolyline() # pick random segment if len(vertices) == 2: vid = 0 else: vid = random.randint(0, len(vertices) - 2) startPoint = vertices[vid] endPoint = vertices[vid + 1] length = da.measureLine(startPoint, endPoint) dist = length * random.random() if dist > minDistance: d = dist / (length - dist) rx = (startPoint.x() + d * endPoint.x()) / (1 + d) ry = (startPoint.y() + d * endPoint.y()) / (1 + d) # generate random point pnt = QgsPoint(rx, ry) geom = QgsGeometry.fromPoint(pnt) if vector.checkMinDistance(pnt, index, minDistance, points): f = QgsFeature(nPoints) f.initAttributes(1) f.setFields(fields) f.setAttribute('id', nPoints) f.setGeometry(geom) writer.addFeature(f) index.insertFeature(f) points[nPoints] = pnt nPoints += 1 feedback.setProgress(int(nPoints * total)) nIterations += 1 if nPoints < pointCount: ProcessingLog.addToLog( ProcessingLog.LOG_INFO, self.tr('Can not generate requested number of random points. ' 'Maximum number of attempts exceeded.')) del writer
def testWriteShapefileWithZ(self): """Check writing geometries with Z dimension to an ESRI shapefile.""" # start by saving a memory layer and forcing z ml = QgsVectorLayer(('Point?crs=epsg:4326&field=id:int'), 'test', 'memory') self.assertIsNotNone(ml, 'Provider not initialized') self.assertTrue(ml.isValid(), 'Source layer not valid') provider = ml.dataProvider() self.assertIsNotNone(provider) ft = QgsFeature() ft.setGeometry(QgsGeometry.fromWkt('PointZ (1 2 3)')) ft.setAttributes([1]) res, features = provider.addFeatures([ft]) self.assertTrue(res) self.assertTrue(features) # check with both a standard PointZ and 25d style Point25D type for t in [QgsWkbTypes.PointZ, QgsWkbTypes.Point25D]: dest_file_name = os.path.join( str(QDir.tempPath()), 'point_{}.shp'.format(QgsWkbTypes.displayString(t))) crs = QgsCoordinateReferenceSystem() crs.createFromId(4326, QgsCoordinateReferenceSystem.EpsgCrsId) write_result, error_message = QgsVectorFileWriter.writeAsVectorFormat( ml, dest_file_name, 'utf-8', crs, 'ESRI Shapefile', overrideGeometryType=t) self.assertEqual(write_result, QgsVectorFileWriter.NoError, error_message) # Open result and check created_layer = QgsVectorLayer( '{}|layerid=0'.format(dest_file_name), 'test', 'ogr') f = next(created_layer.getFeatures(QgsFeatureRequest())) g = f.geometry() wkt = g.asWkt() expWkt = 'PointZ (1 2 3)' self.assertTrue( compareWkt(expWkt, wkt), "saving geometry with Z failed: mismatch Expected:\n%s\nGot:\n%s\n" % (expWkt, wkt)) # also try saving out the shapefile version again, as an extra test # this tests that saving a layer with z WITHOUT explicitly telling the writer to keep z values, # will stay retain the z values dest_file_name = os.path.join( str(QDir.tempPath()), 'point_{}_copy.shp'.format(QgsWkbTypes.displayString(t))) crs = QgsCoordinateReferenceSystem() crs.createFromId(4326, QgsCoordinateReferenceSystem.EpsgCrsId) write_result, error_message = QgsVectorFileWriter.writeAsVectorFormat( created_layer, dest_file_name, 'utf-8', crs, 'ESRI Shapefile') self.assertEqual(write_result, QgsVectorFileWriter.NoError, error_message) # Open result and check created_layer_from_shp = QgsVectorLayer( '{}|layerid=0'.format(dest_file_name), 'test', 'ogr') f = next(created_layer_from_shp.getFeatures(QgsFeatureRequest())) g = f.geometry() wkt = g.asWkt() self.assertTrue( compareWkt(expWkt, wkt), "saving geometry with Z failed: mismatch Expected:\n%s\nGot:\n%s\n" % (expWkt, wkt))
def addFilesToWMBSInBulk(filesetId, workflowName, files, isDBS=True, conn=None, transaction=None): """ _addFilesToWMBSInBulk Do a bulk addition of files into WMBS. This is a speedup. Assumes files are full dao objects """ if not files: # Nothing to do return 0 daofactory = next(iter(files)).daofactory setParentage = daofactory(classname="Files.SetParentage") setFileRunLumi = daofactory(classname="Files.AddRunLumi") setFileLocation = daofactory(classname="Files.SetLocationForWorkQueue") setFileAddChecksum = daofactory(classname="Files.AddChecksumByLFN") addFileAction = daofactory(classname="Files.Add") addToFileset = daofactory(classname="Files.AddDupsToFileset") updateFileAction = daofactory(classname="Files.Update") # build up list of binds for all files then run in single transaction parentageBinds = [] runLumiBinds = [] fileCksumBinds = [] fileLocations = [] fileCreate = [] fileLFNs = set() lfnsToCreate = set() lfnList = set() fileUpdate = [] for wmbsFile in files: lfn = wmbsFile['lfn'] lfnList.add(lfn) if wmbsFile.get('inFileset', True): fileLFNs.add(lfn) for parent in wmbsFile['parents']: parentageBinds.append({'child': lfn, 'parent': parent["lfn"]}) selfChecksums = wmbsFile['checksums'] if len(wmbsFile['runs']) > 0: runLumiBinds.append({'lfn': lfn, 'runs': wmbsFile['runs']}) if len(wmbsFile['newlocations']) < 1: # Then we're in trouble msg = "File created in WMBS without locations!\n" msg += "File lfn: %s\n" % (lfn) logging.error(msg) raise RuntimeError(msg) for loc in wmbsFile['newlocations']: fileLocations.append({'lfn': lfn, 'location': loc}) if wmbsFile.exists(): # update events, size, first_event, merged fileUpdate.append([ lfn, wmbsFile['size'], wmbsFile['events'], None, wmbsFile["first_event"], wmbsFile['merged'] ]) continue lfnsToCreate.add(lfn) if selfChecksums: # If we have checksums we have to create a bind # For each different checksum for entry in selfChecksums: fileCksumBinds.append({ 'lfn': lfn, 'cksum': selfChecksums[entry], 'cktype': entry }) fileCreate.append([ lfn, wmbsFile['size'], wmbsFile['events'], None, wmbsFile["first_event"], wmbsFile['merged'] ]) if len(fileCreate) > 0: addFileAction.execute(files=fileCreate, conn=conn, transaction=transaction) setFileAddChecksum.execute(bulkList=fileCksumBinds, conn=conn, transaction=transaction) if len(fileUpdate) > 0: updateFileAction.execute(files=fileUpdate, conn=conn, transaction=transaction) if len(fileLocations) > 0: setFileLocation.execute(lfns=lfnList, locations=fileLocations, isDBS=isDBS, conn=conn, transaction=transaction) if len(runLumiBinds) > 0: setFileRunLumi.execute(file=runLumiBinds, conn=conn, transaction=transaction) if len(fileLFNs) > 0: addToFileset.execute(file=fileLFNs, fileset=filesetId, workflow=workflowName, conn=conn, transaction=transaction) if len(parentageBinds) > 0: setParentage.execute(binds=parentageBinds, conn=conn, transaction=transaction) return len(lfnsToCreate)
def cli(): logging.basicConfig(level=logging.DEBUG) # Control-T handling hidden = [True] # Nonlocal key_bindings = KeyBindings() @key_bindings.add('c-t') def _(event): ' When Control-T has been pressed, toggle visibility. ' hidden[0] = not hidden[0] def prompt_pass(): print('Type Control-T to toggle password visible.') password = prompt('Password/Key: ', is_password=Condition(lambda: hidden[0]), key_bindings=key_bindings) return password def prompt_key(): print('Type Control-T to toggle key visible.') key = prompt('Key: ', is_password=Condition(lambda: hidden[0]), key_bindings=key_bindings) return key def prompt_pin(): print('Press any key when finished entering PIN') return if len(sys.argv) > 1: if sys.argv[1] == 'settime': only_key.set_time(time.time()) elif sys.argv[1] == 'init': while 1: if only_key.read_string(timeout_ms=500) != 'UNINITIALIZED': break for msg in [Message.OKSETPIN]: only_key.send_message(msg=msg) print(only_key.read_string()) print() input('Press the Enter key once you are done') only_key.send_message(msg=msg) print(only_key.read_string()) only_key.send_message(msg=msg) print(only_key.read_string()) print() input('Press the Enter key once you are done') only_key.send_message(msg=msg) print(only_key.read_string()) print() for msg in [Message.OKSETPDPIN]: only_key.send_message(msg=msg) print(only_key.read_string() + ' for second profile') print() input('Press the Enter key once you are done') only_key.send_message(msg=msg) print(only_key.read_string() + ' for second profile') only_key.send_message(msg=msg) print(only_key.read_string()) print() input('Press the Enter key once you are done') only_key.send_message(msg=msg) print(only_key.read_string()) print() for msg in [Message.OKSETSDPIN]: only_key.send_message(msg=msg) print(only_key.read_string()) print() input('Press the Enter key once you are done') only_key.send_message(msg=msg) print(only_key.read_string()) only_key.send_message(msg=msg) print(only_key.read_string()) print() input('Press the Enter key once you are done') only_key.send_message(msg=msg) print(only_key.read_string()) print() elif sys.argv[1] == 'getlabels': tmp = {} for slot in only_key.getlabels(): tmp[slot.name] = slot slots = iter([ '1a', '1b', '2a', '2b', '3a', '3b', '4a', '4b', '5a', '5b', '6a', '6b' ]) for slot_name in slots: print(tmp[slot_name].to_str().replace('ÿ', " ")) print(tmp[next(slots)].to_str().replace('ÿ', " ")) print() elif sys.argv[1] == 'getkeylabels': tmp = {} for slot in only_key.getkeylabels(): tmp[slot.name] = slot slots = iter([ 'RSA Key 1', 'RSA Key 2', 'RSA Key 3', 'RSA Key 4', 'ECC Key 1', 'ECC Key 2', 'ECC Key 3', 'ECC Key 4', 'ECC Key 5', 'ECC Key 6', 'ECC Key 7', 'ECC Key 8', 'ECC Key 9', 'ECC Key 10', 'ECC Key 11', 'ECC Key 12', 'ECC Key 13', 'ECC Key 14', 'ECC Key 15', 'ECC Key 16' ]) for slot_name in slots: print(tmp[slot_name].to_str().replace('ÿ', " ")) elif sys.argv[1] == 'setslot': try: if sys.argv[2] == '1a': slot_id = 1 elif sys.argv[2] == '2a': slot_id = 2 elif sys.argv[2] == '3a': slot_id = 3 elif sys.argv[2] == '4a': slot_id = 4 elif sys.argv[2] == '5a': slot_id = 5 elif sys.argv[2] == '6a': slot_id = 6 elif sys.argv[2] == '1b': slot_id = 7 elif sys.argv[2] == '2b': slot_id = 8 elif sys.argv[2] == '3b': slot_id = 9 elif sys.argv[2] == '4b': slot_id = 10 elif sys.argv[2] == '5b': slot_id = 11 elif sys.argv[2] == '6b': slot_id = 12 elif sys.argv[2] >= int('25'): slot_id = int(sys.argv[2]) except: print("setslot <id> <type> [value]") print("<id> must be slot number 1a - 6b") return if sys.argv[3] == 'label': only_key.setslot(slot_id, MessageField.LABEL, sys.argv[4]) elif sys.argv[3] == 'ecc_key_label': only_key.setslot(slot_id + 28, MessageField.LABEL, sys.argv[4]) elif sys.argv[3] == 'rsa_key_label': only_key.setslot(slot_id + 24, MessageField.LABEL, sys.argv[4]) elif sys.argv[3] == 'url': only_key.setslot(slot_id, MessageField.URL, sys.argv[4]) elif sys.argv[3] == 'add_char2': only_key.setslot(slot_id, MessageField.NEXTKEY1, sys.argv[4]) elif sys.argv[3] == 'delay1': only_key.setslot(slot_id, MessageField.DELAY1, sys.argv[4]) elif sys.argv[3] == 'username': only_key.setslot(slot_id, MessageField.USERNAME, sys.argv[4]) elif sys.argv[3] == 'add_char3': only_key.setslot(slot_id, MessageField.NEXTKEY2, sys.argv[4]) elif sys.argv[3] == 'delay2': only_key.setslot(slot_id, MessageField.DELAY2, sys.argv[4]) elif sys.argv[3] == 'password': password = prompt_pass() only_key.setslot(slot_id, MessageField.PASSWORD, password) elif sys.argv[3] == 'add_char5': only_key.setslot(slot_id, MessageField.NEXTKEY3, sys.argv[4]) elif sys.argv[3] == 'delay3': only_key.setslot(slot_id, MessageField.DELAY3, sys.argv[4]) elif sys.argv[3] == '2fa': only_key.setslot(slot_id, MessageField.TFATYPE, sys.argv[4]) elif sys.argv[3] == 'gkey': totpkey = prompt_key() totpkey = base64.b32decode("".join(totpkey.split()).upper()) totpkey = binascii.hexlify(totpkey) # pad with zeros for even digits totpkey = totpkey.zfill(len(totpkey) + len(totpkey) % 2) payload = [ int(totpkey[i:i + 2], 16) for i in range(0, len(totpkey), 2) ] only_key.setslot(slot_id, MessageField.TOTPKEY, payload) elif sys.argv[3] == 'totpkey': totpkey = prompt_key() only_key.setslot(slot_id, MessageField.TOTPKEY, totpkey) elif sys.argv[3] == 'add_char1': only_key.setslot(slot_id, MessageField.NEXTKEY4, sys.argv[4]) elif sys.argv[3] == 'add_char4': only_key.setslot(slot_id, MessageField.NEXTKEY5, sys.argv[4]) else: print("setslot <id> <type> [value]") print( "<type> must be ['label', 'ecc_key_label', 'rsa_key_label', 'url', 'add_char1', 'delay1', 'username', 'add_char2', 'delay2', 'password', 'add_char3', 'delay3', '2fa', 'totpkey', 'add_char4', 'add_char5']" ) return elif sys.argv[1] == 'wipeslot': try: if sys.argv[2] == '1a': slot_id = 1 elif sys.argv[2] == '2a': slot_id = 2 elif sys.argv[2] == '3a': slot_id = 3 elif sys.argv[2] == '4a': slot_id = 4 elif sys.argv[2] == '5a': slot_id = 5 elif sys.argv[2] == '6a': slot_id = 6 elif sys.argv[2] == '1b': slot_id = 7 elif sys.argv[2] == '2b': slot_id = 8 elif sys.argv[2] == '3b': slot_id = 9 elif sys.argv[2] == '4b': slot_id = 10 elif sys.argv[2] == '5b': slot_id = 11 elif sys.argv[2] == '6b': slot_id = 12 elif sys.argv[2] >= int('25'): slot_id = int(sys.argv[2]) except: print("wipeslot <id>") print("<id> must be slot number 1a - 6b") return only_key.wipeslot(slot_id) elif sys.argv[1] == 'backupkey': only_key.generate_backup_key() elif sys.argv[1] == 'setkey': only_key.setkey(sys.argv[2], sys.argv[3], sys.argv[4]) elif sys.argv[1] == 'wipekey': only_key.wipekey(sys.argv[2]) elif sys.argv[1] == 'idletimeout': only_key.setslot(1, MessageField.IDLETIMEOUT, int(sys.argv[2])) elif sys.argv[1] == 'wipemode': only_key.setslot(1, MessageField.WIPEMODE, int(sys.argv[2])) elif sys.argv[1] == 'keytypespeed': only_key.setslot(1, MessageField.KEYTYPESPEED, int(sys.argv[2])) elif sys.argv[1] == 'led_brightness': only_key.setslot(1, MessageField.LEDBRIGHTNESS, int(sys.argv[2])) elif sys.argv[1] == '2nd_profile_mode': only_key.setslot(1, MessageField.SECPROFILEMODE, int(sys.argv[2])) elif sys.argv[1] == 'pgp_challenge_mode': only_key.setslot(1, MessageField.PGPCHALENGEMODE, int(sys.argv[2])) elif sys.argv[1] == 'ssh_challenge_mode': only_key.setslot(1, MessageField.SSHCHALENGEMODE, int(sys.argv[2])) elif sys.argv[1] == 'backup_key_mode': only_key.setslot(1, MessageField.BACKUPMODE, int(sys.argv[2])) elif sys.argv[1] == 'keylayout': only_key.setslot(1, MessageField.KEYLAYOUT, int(sys.argv[2])) elif sys.argv[1]: print('Command not found') print() else: # Print help. print('OnlyKey CLI v1.2.2') print('Control-D to exit.') print() def mprompt(): return prompt('OnlyKey> ') nexte = mprompt while 1: print() raw = nexte() print() data = raw.split() # nexte = prompt_pass if data[0] == "settime": only_key.set_time(time.time()) if data[0] == "init": while 1: if only_key.read_string(timeout_ms=500) != 'UNINITIALIZED': break for msg in [Message.OKSETPIN]: only_key.send_message(msg=msg) print(only_key.read_string()) print() input('Press the Enter key once you are done') only_key.send_message(msg=msg) print(only_key.read_string()) only_key.send_message(msg=msg) print(only_key.read_string()) print() input('Press the Enter key once you are done') only_key.send_message(msg=msg) print(only_key.read_string()) print() for msg in [Message.OKSETPDPIN]: only_key.send_message(msg=msg) print(only_key.read_string() + ' for second profile') print() input('Press the Enter key once you are done') only_key.send_message(msg=msg) print(only_key.read_string() + ' for second profile') only_key.send_message(msg=msg) print(only_key.read_string()) print() input('Press the Enter key once you are done') only_key.send_message(msg=msg) print(only_key.read_string()) print() for msg in [Message.OKSETSDPIN]: only_key.send_message(msg=msg) print(only_key.read_string()) print() input('Press the Enter key once you are done') only_key.send_message(msg=msg) print(only_key.read_string()) only_key.send_message(msg=msg) print(only_key.read_string()) print() input('Press the Enter key once you are done') only_key.send_message(msg=msg) print(only_key.read_string()) print() elif data[0] == 'getlabels': tmp = {} for slot in only_key.getlabels(): tmp[slot.name] = slot slots = iter([ '1a', '1b', '2a', '2b', '3a', '3b', '4a', '4b', '5a', '5b', '6a', '6b' ]) for slot_name in slots: print(tmp[slot_name].to_str().replace('ÿ', " ")) print(tmp[next(slots)].to_str().replace('ÿ', " ")) print() if data[0] == 'getkeylabels': tmp = {} for slot in only_key.getkeylabels(): tmp[slot.name] = slot slots = iter([ 'RSA Key 1', 'RSA Key 2', 'RSA Key 3', 'RSA Key 4', 'ECC Key 1', 'ECC Key 2', 'ECC Key 3', 'ECC Key 4', 'ECC Key 5', 'ECC Key 6', 'ECC Key 7', 'ECC Key 8', 'ECC Key 9', 'ECC Key 10', 'ECC Key 11', 'ECC Key 12', 'ECC Key 13', 'ECC Key 14', 'ECC Key 15', 'ECC Key 16' ]) for slot_name in slots: print(tmp[slot_name].to_str().replace('ÿ', " ")) elif data[0] == 'setslot': try: if data[1] == '1a': slot_id = 1 elif data[1] == '2a': slot_id = 2 elif data[1] == '3a': slot_id = 3 elif data[1] == '4a': slot_id = 4 elif data[1] == '5a': slot_id = 5 elif data[1] == '6a': slot_id = 6 elif data[1] == '1b': slot_id = 7 elif data[1] == '2b': slot_id = 8 elif data[1] == '3b': slot_id = 9 elif data[1] == '4b': slot_id = 10 elif data[1] == '5b': slot_id = 11 elif data[1] == '6b': slot_id = 12 elif data[1] >= int('25'): slot_id = int(data[1]) except: print("setslot <id> <type> [value]") print("<id> must be slot number 1a - 6b") continue if data[2] == 'label': only_key.setslot(slot_id, MessageField.LABEL, data[3]) elif data[2] == 'ecc_key_label': only_key.setslot(slot_id + 28, MessageField.LABEL, data[3]) elif data[2] == 'rsa_key_label': only_key.setslot(slot_id + 24, MessageField.LABEL, data[3]) elif data[2] == 'url': only_key.setslot(slot_id, MessageField.URL, data[3]) elif data[2] == 'add_char2': only_key.setslot(slot_id, MessageField.NEXTKEY1, data[3]) elif data[2] == 'delay1': only_key.setslot(slot_id, MessageField.DELAY1, data[3]) elif data[2] == 'username': only_key.setslot(slot_id, MessageField.USERNAME, data[3]) elif data[2] == 'add_char3': only_key.setslot(slot_id, MessageField.NEXTKEY2, data[3]) elif data[2] == 'delay2': only_key.setslot(slot_id, MessageField.DELAY2, data[3]) elif data[2] == 'password': password = prompt_pass() only_key.setslot(slot_id, MessageField.PASSWORD, password) elif data[2] == 'add_char5': only_key.setslot(slot_id, MessageField.NEXTKEY3, data[3]) elif data[2] == 'delay3': only_key.setslot(slot_id, MessageField.DELAY3, data[3]) elif data[2] == '2fa': only_key.setslot(slot_id, MessageField.TFATYPE, data[3]) elif data[2] == 'gkey': totpkey = prompt_key() totpkey = base64.b32decode("".join( totpkey.split()).upper()) totpkey = binascii.hexlify(totpkey) # pad with zeros for even digits totpkey = totpkey.zfill(len(totpkey) + len(totpkey) % 2) payload = [ int(totpkey[i:i + 2], 16) for i in range(0, len(totpkey), 2) ] only_key.setslot(slot_id, MessageField.TOTPKEY, payload) elif data[2] == 'totpkey': totpkey = prompt_key() only_key.setslot(slot_id, MessageField.TOTPKEY, totpkey) elif data[2] == 'add_char1': only_key.setslot(slot_id, MessageField.NEXTKEY3, data[3]) elif data[2] == 'add_char4': only_key.setslot(slot_id, MessageField.NEXTKEY3, data[3]) else: print("setslot <id> <type> [value]") print( "<type> must be ['label', 'ecc_key_label', 'rsa_key_label', 'url', 'add_char1', 'delay1', 'username', 'add_char2', 'delay2', 'password', 'add_char3', 'delay3', '2fa', 'totpkey', 'add_char4', 'add_char5']" ) continue elif data[0] == 'wipeslot': try: if data[1] == '1a': slot_id = 1 elif data[1] == '2a': slot_id = 2 elif data[1] == '3a': slot_id = 3 elif data[1] == '4a': slot_id = 4 elif data[1] == '5a': slot_id = 5 elif data[1] == '6a': slot_id = 6 elif data[1] == '1b': slot_id = 7 elif data[1] == '2b': slot_id = 8 elif data[1] == '3b': slot_id = 9 elif data[1] == '4b': slot_id = 10 elif data[1] == '5b': slot_id = 11 elif data[1] == '6b': slot_id = 12 elif data[1] >= int('25'): slot_id = int(data[1]) except: print("wipeslot <id>") print("<id> must be slot number 1a - 6b") continue only_key.wipeslot(slot_id) elif data[0] == 'backupkey': try: only_key.generate_backup_key() except: continue elif data[0] == 'setkey': try: key = prompt_pass() only_key.setkey(data[1], data[2], key) except: continue elif data[0] == 'wipekey': try: only_key.wipekey(data[1]) except: continue elif data[0] == 'idletimeout': try: only_key.setslot(1, MessageField.IDLETIMEOUT, int(data[1])) except: continue elif data[0] == 'wipemode': try: only_key.setslot(1, MessageField.WIPEMODE, int(data[1])) except: continue elif data[0] == 'keytypespeed': try: only_key.setslot(1, MessageField.KEYTYPESPEED, int(data[1])) except: continue elif data[0] == 'led_brightness': try: only_key.setslot(1, MessageField.LEDBRIGHTNESS, int(data[1])) except: continue elif data[0] == 'pgp_challenge_mode': try: only_key.setslot(1, MessageField.PGPCHALENGEMODE, int(data[1])) except: continue elif data[0] == 'ssh_challenge_mode': try: only_key.setslot(1, MessageField.SSHCHALENGEMODE, int(data[1])) except: continue elif data[0] == 'backup_key_mode': try: only_key.setslot(1, MessageField.BACKUPMODE, int(data[1])) except: continue elif data[0] == '2nd_profile_mode': try: only_key.setslot(1, MessageField.SECPROFILEMODE, int(data[1])) except: continue elif data[0] == 'keylayout': try: only_key.setslot(1, MessageField.KEYLAYOUT, int(data[1])) except: continue
def test_http_operate_tests_async(): with InteractivePlan( name="InteractivePlan", interactive_port=0, interactive_block=False, parse_cmdline=False, logger_level=TEST_INFO, ) as plan: plan.run() wait(lambda: any(plan.i.http_handler_info), 5, raise_on_timeout=True) addr = "http://{}:{}".format(*plan.i.http_handler_info) plan.add(make_multitest(1)) plan.add(make_multitest(2)) # TRIGGER ASYNC RUN OF TESTS -> UID response = post_request("{}/async/run_tests".format(addr), {}).json() expected = { "message": "Async operation performed: run_tests", "error": False, "trace": None, "metadata": {}, "result": re.compile("[0-9|a-z|-]+"), } assert compare(expected, response)[0] is True uid = response["result"] # QUERY UID ASYNC OPERATION UNTIL FINISHED sleeper = get_sleeper(0.6, raise_timeout_with_msg="Async result missing.") while next(sleeper): response = post_request("{}/async_result".format(addr), {"uid": uid}) json_response = response.json() if json_response["error"] is False: assert response.status_code == 200 expected = { "result": None, "trace": None, "error": False, "message": re.compile("[0-9|a-z|-]+"), "metadata": { "state": "Finished" }, } assert compare(expected, json_response)[0] is True break assert response.status_code == 400 # REPORT VIA HTTP response = post_request("{}/sync/report".format(addr), { "serialized": True }).json() expected_response = { "result": plan.i.report(serialized=True), "error": False, "metadata": {}, "trace": None, "message": "Sync operation performed: report", } assert compare(response, expected_response)[0] is True
def subcellularConn(self, allCellTags, allPopTags): """ Function for/to <short description of `netpyne.network.subconn.subcellularConn`> Parameters ---------- self : <type> <Short description of self> **Default:** *required* allCellTags : <type> <Short description of allCellTags> **Default:** *required* allPopTags : <type> <Short description of allPopTags> **Default:** *required* """ from .. import sim sim.timing('start', 'subConnectTime') print(' Distributing synapses based on subcellular connectivity rules...') for subConnParamTemp in list(self.params.subConnParams.values() ): # for each conn rule or parameter set subConnParam = subConnParamTemp.copy() # find list of pre and post cell preCellsTags, postCellsTags = self._findPrePostCellsCondition( allCellTags, subConnParam['preConds'], subConnParam['postConds']) if preCellsTags and postCellsTags: # iterate over postsyn cells to redistribute synapses for postCellGid in postCellsTags: # for each postsyn cell if postCellGid in self.gid2lid: postCell = self.cells[self.gid2lid[postCellGid]] allConns = [ conn for conn in postCell.conns if conn['preGid'] in preCellsTags ] if 'NetStim' in [ x['cellModel'] for x in list(preCellsTags.values()) if 'cellModel' in x.keys() ]: # temporary fix to include netstim conns allConns.extend([ conn for conn in postCell.conns if conn['preGid'] == 'NetStim' ]) # group synMechs so they are not distributed separately if 'groupSynMechs' in subConnParam and len( subConnParam['groupSynMechs']) > 1: conns = [] connsGroup = {} #iConn = -1 for conn in allConns: if not conn['synMech'].startswith('__grouped__'): conns.append(conn) #iConn = iConn + 1 connGroupLabel = '%d_%s_%.4f' % ( conn['preGid'], conn['sec'], conn['loc']) if conn['synMech'] in subConnParam[ 'groupSynMechs']: for synMech in [ s for s in subConnParam['groupSynMechs'] if s != conn['synMech'] ]: connGroup = next( (c for c in allConns if c['synMech'] == synMech and c['sec'] == conn['sec'] and c['loc'] == conn['loc']), None) try: connGroup[ 'synMech'] = '__grouped__' + connGroup[ 'synMech'] connsGroup[ connGroupLabel] = connGroup except: print( ' Warning: Grouped synMechs %s not found' % (str(connGroup))) else: conns = allConns # sort conns so reproducible across different number of cores # use sec+preGid to avoid artificial distribution based on preGid (e.g. low gids = close to soma) conns = sorted(conns, key=lambda v: v['sec'] + str(v['loc']) + str(v['preGid'])) # set sections to be used secList = postCell._setConnSections(subConnParam) # Uniform distribution if subConnParam.get('density', None) == 'uniform': # calculate new syn positions newSecs, newLocs = postCell._distributeSynsUniformly( secList=secList, numSyns=len(conns)) # 2D map and 1D map (radial) elif isinstance( subConnParam.get('density', None), dict) and subConnParam['density']['type'] in [ '2Dmap', '1Dmap' ]: gridY = subConnParam['density']['gridY'] gridSigma = subConnParam['density']['gridValues'] somaX, somaY, _ = self._posFromLoc( postCell.secs['soma']['hObj'], 0.5) # get cell pos move method to Cell! if 'fixedSomaY' in subConnParam[ 'density']: # is fixed cell soma y, adjust y grid accordingly fixedSomaY = subConnParam['density'].get( 'fixedSomaY') gridY = [ y + (somaY - fixedSomaY) for y in gridY ] # adjust grid so cell soma is at fixedSomaY if subConnParam['density']['type'] == '2Dmap': # 2D gridX = [ x - somaX for x in subConnParam['density']['gridX'] ] # center x at cell soma segNumSyn = self._interpolateSegmentSigma( postCell, secList, gridX, gridY, gridSigma) # move method to Cell! elif subConnParam['density']['type'] == '1Dmap': # 1D segNumSyn = self._interpolateSegmentSigma( postCell, secList, None, gridY, gridSigma) # move method to Cell! totSyn = sum([ sum(nsyn) for nsyn in list(segNumSyn.values()) ]) # summed density scaleNumSyn = float( len(conns)) / float(totSyn) if totSyn > 0 else 0.0 diffList = [] for sec in segNumSyn: for seg, x in enumerate(segNumSyn[sec]): orig = float(x * scaleNumSyn) scaled = int(round(x * scaleNumSyn)) segNumSyn[sec][seg] = scaled diff = orig - scaled if diff > 0: diffList.append([diff, sec, seg]) totSynRescale = sum( [sum(nsyn) for nsyn in list(segNumSyn.values())]) # if missing syns due to rescaling to 0, find top values which were rounded to 0 and make 1 if totSynRescale < len(conns): extraSyns = len(conns) - totSynRescale diffList = sorted(diffList, key=lambda l: l[0], reverse=True) for i in range(min(extraSyns, len(diffList))): sec = diffList[i][1] seg = diffList[i][2] segNumSyn[sec][seg] += 1 # convert to list so can serialize and save subConnParam['density']['gridY'] = list( subConnParam['density']['gridY']) subConnParam['density']['gridValues'] = list( subConnParam['density']['gridValues']) newSecs, newLocs = [], [] for sec, nsyns in segNumSyn.items(): for i, seg in enumerate( postCell.secs[sec]['hObj']): for isyn in range(nsyns[i]): newSecs.append(sec) newLocs.append(seg.x) # Distance-based elif isinstance( subConnParam.get('density', None), dict ) and subConnParam['density']['type'] == 'distance': # find origin section # default if 'soma' in postCell.secs: secOrig = 'soma' elif any([ secName.startswith('som') for secName in list(postCell.secs.keys()) ]): secOrig = next( secName for secName in list(postCell.secs.keys()) if secName.startswith('soma')) else: secOrig = list(postCell.secs.keys())[0] # giving argument if 'ref_sec' in subConnParam['density']: if subConnParam['density']['ref_sec'] in list( postCell.secs.keys()): secOrig = subConnParam['density']['ref_sec'] else: print( ' Warning: Redistributing synapses based on inexistent information for neuron %d - section %s not found' % (postCell.gid, subConnParam['density']['ref_sec'])) # find origin segment segOrig = 0.5 # default if 'ref_seg' in subConnParam['density']: segOrig = subConnParam['density']['ref_seg'] # target target_distance = 0.0 if 'target_distance' in subConnParam['density']: target_distance = subConnParam['density'][ 'target_distance'] newSec, newLoc = secOrig, segOrig min_dist = target_distance if 'coord' in subConnParam['density'] and subConnParam[ 'density']['coord'] == 'cartesian': # calculate euclidean distance from reference x0, y0, z0 = self._posFromLoc( postCell.secs[secOrig]['hObj'], postCell.secs[secOrig]['hObj'](segOrig).x) for secName in secList: for seg in postCell.secs[secName]['hObj']: x, y, z = self._posFromLoc( postCell.secs[secName]['hObj'], seg.x) dist = np.sqrt((x - x0)**2 + (y - y0)**2 + (z - z0)**2) if abs(dist - target_distance) <= min_dist: min_dist = abs(dist - target_distance) newSec, newLoc = secName, seg.x else: # (default) calculate distance based on the topology for secName in secList: for seg in postCell.secs[secName]['hObj']: dist = self.fromtodistance( postCell.secs[secOrig]['hObj']( segOrig), seg) if abs(dist - target_distance) <= min_dist: min_dist = abs(dist - target_distance) newSec, newLoc = secName, seg.x newSecs = [newSec] * len(conns) newLocs = [newLoc] * len(conns) for i, (conn, newSec, newLoc) in enumerate(zip(conns, newSecs, newLocs)): # get conn group label before updating params connGroupLabel = '%d_%s_%.4f' % ( conn['preGid'], conn['sec'], conn['loc']) # update weight if weightNorm present newWeightNorm = None if 'weightNorm' in postCell.secs[ conn['sec']] and isinstance( postCell.secs[conn['sec']]['weightNorm'], list): oldNseg = postCell.secs[ conn['sec']]['geom']['nseg'] oldWeightNorm = postCell.secs[ conn['sec']]['weightNorm'][ int(round(conn['loc'] * oldNseg)) - 1] newNseg = postCell.secs[newSec]['geom']['nseg'] newWeightNorm = postCell.secs[newSec]['weightNorm'][ int(round(newLoc * newNseg)) - 1] if 'weightNorm' in postCell.secs[ newSec] else 1.0 conn['weight'] = conn[ 'weight'] / oldWeightNorm * newWeightNorm # avoid locs at 0.0 or 1.0 - triggers hoc error if syn needs an ion (eg. ca_ion) if newLoc == 0.0: newLoc = 0.00001 elif newLoc == 1.0: newLoc = 0.99999 # updade sec and loc conn['sec'] = newSec conn['loc'] = newLoc # find grouped conns if subConnParam.get('groupSynMechs', None) \ and len(subConnParam['groupSynMechs']) > 1 \ and conn['synMech'] in subConnParam['groupSynMechs']: connGroup = connsGroup[ connGroupLabel] # get grouped conn from previously stored dict connGroup['synMech'] = connGroup['synMech'].split( '__grouped__')[1] # remove '__grouped__' label connGroup['sec'] = newSec connGroup['loc'] = newLoc if newWeightNorm: connGroup['weight'] = connGroup[ 'weight'] / oldWeightNorm * newWeightNorm sim.pc.barrier()
def voronoi(siteList, context): try: edgeList = EdgeList(siteList.xmin, siteList.xmax, len(siteList)) priorityQ = PriorityQueue(siteList.ymin, siteList.ymax, len(siteList)) siteIter = siteList.iterator() bottomsite = next(siteIter) context.outSite(bottomsite) newsite = next(siteIter) minpt = Site(-BIG_FLOAT, -BIG_FLOAT) while True: if not priorityQ.isEmpty(): minpt = priorityQ.getMinPt() if (newsite and (priorityQ.isEmpty() or cmp(newsite, minpt) < 0)): # newsite is smallest - this is a site event context.outSite(newsite) # get first Halfedge to the LEFT and RIGHT of the new site lbnd = edgeList.leftbnd(newsite) rbnd = lbnd.right # if this halfedge has no edge, bot = bottom site (whatever that is) # create a new edge that bisects bot = lbnd.rightreg(bottomsite) edge = Edge.bisect(bot, newsite) context.outBisector(edge) # create a new Halfedge, setting its pm field to 0 and insert # this new bisector edge between the left and right vectors in # a linked list bisector = Halfedge(edge, Edge.LE) edgeList.insert(lbnd, bisector) # if the new bisector intersects with the left edge, remove # the left edge's vertex, and put in the new one p = lbnd.intersect(bisector) if p is not None: priorityQ.delete(lbnd) priorityQ.insert(lbnd, p, newsite.distance(p)) # create a new Halfedge, setting its pm field to 1 # insert the new Halfedge to the right of the original bisector lbnd = bisector bisector = Halfedge(edge, Edge.RE) edgeList.insert(lbnd, bisector) # if this new bisector intersects with the right Halfedge p = bisector.intersect(rbnd) if p is not None: # push the Halfedge into the ordered linked list of vertices priorityQ.insert(bisector, p, newsite.distance(p)) newsite = next(siteIter) elif not priorityQ.isEmpty(): # intersection is smallest - this is a vector (circle) event # pop the Halfedge with the lowest vector off the ordered list of # vectors. Get the Halfedge to the left and right of the above HE # and also the Halfedge to the right of the right HE lbnd = priorityQ.popMinHalfedge() llbnd = lbnd.left rbnd = lbnd.right rrbnd = rbnd.right # get the Site to the left of the left HE and to the right of # the right HE which it bisects bot = lbnd.leftreg(bottomsite) top = rbnd.rightreg(bottomsite) # output the triple of sites, stating that a circle goes through them mid = lbnd.rightreg(bottomsite) context.outTriple(bot, top, mid) # get the vertex that caused this event and set the vertex number # couldn't do this earlier since we didn't know when it would be processed v = lbnd.vertex siteList.setSiteNumber(v) context.outVertex(v) # set the endpoint of the left and right Halfedge to be this vector if lbnd.edge.setEndpoint(lbnd.pm, v): context.outEdge(lbnd.edge) if rbnd.edge.setEndpoint(rbnd.pm, v): context.outEdge(rbnd.edge) # delete the lowest HE, remove all vertex events to do with the # right HE and delete the right HE edgeList.delete(lbnd) priorityQ.delete(rbnd) edgeList.delete(rbnd) # if the site to the left of the event is higher than the Site # to the right of it, then swap them and set 'pm' to RIGHT pm = Edge.LE if bot.y > top.y: bot, top = top, bot pm = Edge.RE # Create an Edge (or line) that is between the two Sites. This # creates the formula of the line, and assigns a line number to it edge = Edge.bisect(bot, top) context.outBisector(edge) # create a HE from the edge bisector = Halfedge(edge, pm) # insert the new bisector to the right of the left HE # set one endpoint to the new edge to be the vector point 'v' # If the site to the left of this bisector is higher than the right # Site, then this endpoint is put in position 0; otherwise in pos 1 edgeList.insert(llbnd, bisector) if edge.setEndpoint(Edge.RE - pm, v): context.outEdge(edge) # if left HE and the new bisector don't intersect, then delete # the left HE, and reinsert it p = llbnd.intersect(bisector) if p is not None: priorityQ.delete(llbnd) priorityQ.insert(llbnd, p, bot.distance(p)) # if right HE and the new bisector don't intersect, then reinsert it p = bisector.intersect(rrbnd) if p is not None: priorityQ.insert(bisector, p, bot.distance(p)) else: break he = edgeList.leftend.right while he is not edgeList.rightend: context.outEdge(he.edge) he = he.right Edge.EDGE_NUM = 0 except Exception as err: # fix_print_with_import print("######################################################") # fix_print_with_import print(str(err))
def processAlgorithm(self, progress): layer = dataobjects.getObjectFromUri(self.getParameterValue( self.INPUT)) buf = self.getParameterValue(self.BUFFER) writer = self.getOutputFromName(self.OUTPUT).getVectorWriter( layer.fields().toList(), QgsWkbTypes.Polygon, layer.crs()) outFeat = QgsFeature() extent = layer.extent() extraX = extent.height() * (buf / 100.0) extraY = extent.width() * (buf / 100.0) height = extent.height() width = extent.width() c = voronoi.Context() pts = [] ptDict = {} ptNdx = -1 features = vector.features(layer) total = 100.0 / len(features) for current, inFeat in enumerate(features): geom = inFeat.geometry() point = geom.asPoint() x = point.x() - extent.xMinimum() y = point.y() - extent.yMinimum() pts.append((x, y)) ptNdx += 1 ptDict[ptNdx] = inFeat.id() progress.setPercentage(int(current * total)) if len(pts) < 3: raise GeoAlgorithmExecutionException( self.tr('Input file should contain at least 3 points. Choose ' 'another file and try again.')) uniqueSet = set(item for item in pts) ids = [pts.index(item) for item in uniqueSet] sl = voronoi.SiteList([ voronoi.Site(i[0], i[1], sitenum=j) for (j, i) in enumerate(uniqueSet) ]) voronoi.voronoi(sl, c) inFeat = QgsFeature() current = 0 total = 100.0 / len(c.polygons) for (site, edges) in list(c.polygons.items()): request = QgsFeatureRequest().setFilterFid(ptDict[ids[site]]) inFeat = next(layer.getFeatures(request)) lines = self.clip_voronoi(edges, c, width, height, extent, extraX, extraY) geom = QgsGeometry.fromMultiPoint(lines) geom = QgsGeometry(geom.convexHull()) outFeat.setGeometry(geom) outFeat.setAttributes(inFeat.attributes()) writer.addFeature(outFeat) current += 1 progress.setPercentage(int(current * total)) del writer
def processAlgorithm(self, progress): currentOs = os.name path = OTBUtils.otbPath() commands = [] commands.append(os.path.join(path, self.cliName)) self.roiVectors = {} self.roiRasters = {} for param in self.parameters: # get the given input(s) if param.name in ["-il", "-in"]: newparams = "" listeParameters = param.value.split(";") for inputParameter in listeParameters: # if HDF5 file if "HDF5" in inputParameter: if currentOs == "posix": data = inputParameter[6:] else: data = inputParameter[5:] dataset = data #on windows, there isn't " #if data[-1] == '"': if currentOs == "posix": data = data[:data.index('"')] else: data = data[:data.index('://')] #try : if currentOs == "posix": dataset.index('"') dataset = os.path.basename( data) + dataset[dataset.index('"'):] #except ValueError : else: #dataset = os.path.basename( data ) + '"' + dataset[dataset.index('://'):] dataset = dataset[dataset.index('://'):] #get index of the subdataset with gdal if currentOs == "posix": commandgdal = "gdalinfo " + data + " | grep '" + dataset + "$'" else: commandgdal = "gdalinfo " + data + " | findstr \"" + dataset + "$\"" resultGDAL = os.popen(commandgdal).readlines() indexSubdataset = -1 if resultGDAL: indexSubdatasetString = re.search( "SUBDATASET_(\d+)_", resultGDAL[0]) if indexSubdatasetString: #match between () indexSubdataset = indexSubdatasetString.group( 1) else: indexSubdataset = -1 else: #print "Error : no match of ", dataset, "$ in gdalinfo " + data indexSubdataset = -1 if not indexSubdataset == -1: indexSubdataset = int(indexSubdataset) - 1 newParam = "\'" + data + "?&sdataidx=" + str( indexSubdataset) + "\'" else: newParam = inputParameter newparams += newParam # no hdf5 else: newparams += inputParameter newparams += ";" if newparams[-1] == ";": newparams = newparams[:-1] param.value = newparams if param.value is None or param.value == "": continue if isinstance(param, ParameterVector): commands.append(param.name) if self.hasROI: roiFile = getTempFilename('shp') commands.append(roiFile) self.roiVectors[param.value] = roiFile else: commands.append("\"" + param.value + "\"") elif isinstance(param, ParameterRaster): commands.append(param.name) if self.hasROI: roiFile = getTempFilename('tif') commands.append(roiFile) self.roiRasters[param.value] = roiFile else: commands.append("\"" + param.value + "\"") elif isinstance(param, ParameterMultipleInput): commands.append(param.name) files = str(param.value).split(";") paramvalue = " ".join(["\"" + f + " \"" for f in files]) commands.append(paramvalue) elif isinstance(param, ParameterSelection): commands.append(param.name) idx = int(param.value) commands.append(str(param.options[idx][1])) elif isinstance(param, ParameterBoolean): if param.value: commands.append(param.name) commands.append(str(param.value).lower()) elif isinstance(param, ParameterExtent): self.roiValues = param.value.split(",") else: commands.append(param.name) commands.append(str(param.value)) for out in self.outputs: commands.append(out.name) commands.append('"' + out.value + '"') for roiInput, roiFile in list(self.roiRasters.items()): startX, startY = float(self.roiValues[0]), float(self.roiValues[1]) sizeX = float(self.roiValues[2]) - startX sizeY = float(self.roiValues[3]) - startY helperCommands = [ "otbcli_ExtractROI", "-in", roiInput, "-out", roiFile, "-startx", str(startX), "-starty", str(startY), "-sizex", str(sizeX), "-sizey", str(sizeY) ] ProcessingLog.addToLog(ProcessingLog.LOG_INFO, helperCommands) progress.setCommand(helperCommands) OTBUtils.executeOtb(helperCommands, progress) if self.roiRasters: supportRaster = next(iter(list(self.roiRasters.values()))) for roiInput, roiFile in list(self.roiVectors.items()): helperCommands = [ "otbcli_VectorDataExtractROIApplication", "-vd.in", roiInput, "-io.in", supportRaster, "-io.out", roiFile, "-elev.dem.path", OTBUtils.otbSRTMPath() ] ProcessingLog.addToLog(ProcessingLog.LOG_INFO, helperCommands) progress.setCommand(helperCommands) OTBUtils.executeOtb(helperCommands, progress) loglines = [] loglines.append(self.tr('OTB execution command')) for line in commands: loglines.append(line) progress.setCommand(line) ProcessingLog.addToLog(ProcessingLog.LOG_INFO, loglines) import processing.algs.otb.OTBSpecific_XMLLoading module = processing.algs.otb.OTBSpecific_XMLLoading found = False if 'adapt%s' % self.appkey in dir(module): found = True commands = getattr(module, 'adapt%s' % self.appkey)(commands) else: the_key = 'adapt%s' % self.appkey if '-' in the_key: base_key = the_key.split("-")[0] if base_key in dir(module): found = True commands = getattr(module, base_key)(commands) if not found: ProcessingLog.addToLog( ProcessingLog.LOG_INFO, self.tr("Adapter for %s not found") % the_key) #frames = inspect.getouterframes(inspect.currentframe())[1:] #for a_frame in frames: # frame,filename,line_number,function_name,lines,index = a_frame # ProcessingLog.addToLog(ProcessingLog.LOG_INFO, "%s %s %s %s %s %s" % (frame,filename,line_number,function_name,lines,index)) OTBUtils.executeOtb(commands, progress)
def __next__(self): return next(self._run_list_iter)
def has_with_prefix(self, prefix): """Return whether if there is an item with the given prefix.""" return bool(next(self.get_by_prefix(prefix), None))
def __init__(self, run_list): self._run_list_iter = iter(run_list) self.start, self.end, self.value = next(self)
def range_pair2(facet_field, cat, fq_filter, iterable, end, facet, collection_facet=None): # e.g. counts":["0",17430,"1000",1949,"2000",671,"3000",404,"4000",243,"5000",165],"gap":1000,"start":0,"end":6000} pairs = [] selected_values = [f['value'] for f in fq_filter] is_single_unit_gap = re.match('^[\+\-]?1[A-Za-z]*$', str(facet['gap'])) is not None is_up = facet['sort'] == 'asc' if facet['sort'] == 'asc' and facet['type'] == 'range-up': prev = None n = [] for e in iterable: if prev is not None: n.append(e) n.append(prev) prev = None else: prev = e iterable = n iterable.reverse() a, to = itertools.tee(iterable) next(to, None) counts = iterable[1::2] total_counts = counts.pop(0) if facet['sort'] == 'asc' else 0 sum_all = collection_facet and collection_facet['widgetType'] in ( 'timeline-widget', 'bucket-widget') and facet['type'] == 'range-up' for element in a: next(to, None) to_value = next(to, end) count = next(a) if sum_all: count = total_counts pairs.append({ 'field': facet_field, 'from': element, 'value': count, 'to': to_value, 'selected': element in selected_values, 'exclude': all([f['exclude'] for f in fq_filter if f['value'] == element]), 'is_single_unit_gap': is_single_unit_gap, 'total_counts': total_counts, 'is_up': is_up }) total_counts += counts.pop(0) if counts else 0 if facet['sort'] == 'asc' and facet['type'] != 'range-up': pairs.reverse() return pairs
def processAlgorithm(self, parameters, context, feedback): if parameters[self.INPUT] == parameters[self.HUBS]: raise QgsProcessingException( self.tr('Same layer given for both hubs and spokes')) point_source = self.parameterAsSource(parameters, self.INPUT, context) hub_source = self.parameterAsSource(parameters, self.HUBS, context) fieldName = self.parameterAsString(parameters, self.FIELD, context) units = self.UNITS[self.parameterAsEnum(parameters, self.UNIT, context)] fields = point_source.fields() fields.append(QgsField('HubName', QVariant.String)) fields.append(QgsField('HubDist', QVariant.Double)) (sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, fields, QgsWkbTypes.Point, point_source.sourceCrs()) index = QgsSpatialIndex( hub_source.getFeatures(QgsFeatureRequest().setSubsetOfAttributes( []).setDestinationCrs(point_source.sourceCrs()))) distance = QgsDistanceArea() distance.setSourceCrs(point_source.sourceCrs()) distance.setEllipsoid(context.project().ellipsoid()) # Scan source points, find nearest hub, and write to output file features = point_source.getFeatures() total = 100.0 / point_source.featureCount( ) if point_source.featureCount() else 0 for current, f in enumerate(features): if feedback.isCanceled(): break if not f.hasGeometry(): sink.addFeature(f, QgsFeatureSink.FastInsert) continue src = f.geometry().boundingBox().center() neighbors = index.nearestNeighbor(src, 1) ft = next( hub_source.getFeatures(QgsFeatureRequest().setFilterFid( neighbors[0]).setSubsetOfAttributes( [fieldName], hub_source.fields()).setDestinationCrs( point_source.sourceCrs()))) closest = ft.geometry().boundingBox().center() hubDist = distance.measureLine(src, closest) if units != self.LAYER_UNITS: hub_dist_in_desired_units = distance.convertLengthMeasurement( hubDist, units) else: hub_dist_in_desired_units = hubDist attributes = f.attributes() attributes.append(ft[fieldName]) attributes.append(hub_dist_in_desired_units) feat = QgsFeature() feat.setAttributes(attributes) feat.setGeometry(QgsGeometry.fromPoint(src)) sink.addFeature(feat, QgsFeatureSink.FastInsert) feedback.setProgress(int(current * total)) return {self.OUTPUT: dest_id}
def _get_session(self, notebook, type='hive'): session = next( (session for session in notebook['sessions'] if session['type'] == type), None) return session
def __getitem__(self, index): while index >= self.end and index > self.start: # condition has special case for 0-length run (fixes issue 471) self.start, self.end, self.value = next(self) return self.value
def testWriteShapefileWithAttributeSubsets(self): """Tests writing subsets of attributes to files.""" ml = QgsVectorLayer(( 'Point?crs=epsg:4326&field=id:int&field=field1:int&field=field2:int&field=field3:int' ), 'test', 'memory') self.assertIsNotNone(ml, 'Provider not initialized') self.assertTrue(ml.isValid(), 'Source layer not valid') provider = ml.dataProvider() self.assertIsNotNone(provider) ft = QgsFeature() ft.setGeometry(QgsGeometry.fromWkt('Point (1 2)')) ft.setAttributes([1, 11, 12, 13]) res, features = provider.addFeatures([ft]) self.assertTrue(res) self.assertTrue(features) # first write out with all attributes dest_file_name = os.path.join(str(QDir.tempPath()), 'all_attributes.shp') crs = QgsCoordinateReferenceSystem() crs.createFromId(4326, QgsCoordinateReferenceSystem.EpsgCrsId) write_result, error_message = QgsVectorFileWriter.writeAsVectorFormat( ml, dest_file_name, 'utf-8', crs, 'ESRI Shapefile', attributes=[]) self.assertEqual(write_result, QgsVectorFileWriter.NoError, error_message) # Open result and check created_layer = QgsVectorLayer('{}|layerid=0'.format(dest_file_name), 'test', 'ogr') self.assertEqual(created_layer.fields().count(), 4) f = next(created_layer.getFeatures(QgsFeatureRequest())) self.assertEqual(f['id'], 1) self.assertEqual(f['field1'], 11) self.assertEqual(f['field2'], 12) self.assertEqual(f['field3'], 13) # now test writing out only a subset of attributes dest_file_name = os.path.join(str(QDir.tempPath()), 'subset_attributes.shp') write_result, error_message = QgsVectorFileWriter.writeAsVectorFormat( ml, dest_file_name, 'utf-8', crs, 'ESRI Shapefile', attributes=[1, 3]) self.assertEqual(write_result, QgsVectorFileWriter.NoError, error_message) # Open result and check created_layer = QgsVectorLayer('{}|layerid=0'.format(dest_file_name), 'test', 'ogr') self.assertEqual(created_layer.fields().count(), 2) f = next(created_layer.getFeatures(QgsFeatureRequest())) self.assertEqual(f['field1'], 11) self.assertEqual(f['field3'], 13) # finally test writing no attributes dest_file_name = os.path.join(str(QDir.tempPath()), 'no_attributes.shp') write_result, error_message = QgsVectorFileWriter.writeAsVectorFormat( ml, dest_file_name, 'utf-8', crs, 'ESRI Shapefile', skipAttributeCreation=True) self.assertEqual(write_result, QgsVectorFileWriter.NoError, error_message) # Open result and check created_layer = QgsVectorLayer('{}|layerid=0'.format(dest_file_name), 'test', 'ogr') # expect only a default 'FID' field for shapefiles self.assertEqual(created_layer.fields().count(), 1) self.assertEqual(created_layer.fields()[0].name(), 'FID') # in this case we also check that the geometry exists, to make sure feature has been correctly written # even without attributes f = next(created_layer.getFeatures(QgsFeatureRequest())) g = f.geometry() wkt = g.asWkt() expWkt = 'Point (1 2)' self.assertTrue( compareWkt(expWkt, wkt), "geometry not saved correctly when saving without attributes : mismatch Expected:\n%s\nGot:\n%s\n" % (expWkt, wkt)) self.assertEqual(f['FID'], 0)
def processAlgorithm(self, context, feedback): layer = dataobjects.getLayerFromString( self.getParameterValue(self.INPUT)) fields = [ QgsField('POINTA', QVariant.Double, '', 24, 15), QgsField('POINTB', QVariant.Double, '', 24, 15), QgsField('POINTC', QVariant.Double, '', 24, 15) ] writer = self.getOutputFromName(self.OUTPUT).getVectorWriter( fields, QgsWkbTypes.Polygon, layer.crs(), context) pts = [] ptDict = {} ptNdx = -1 c = voronoi.Context() features = QgsProcessingUtils.getFeatures(layer, context) total = 100.0 / QgsProcessingUtils.featureCount(layer, context) for current, inFeat in enumerate(features): geom = QgsGeometry(inFeat.geometry()) if geom.isNull(): continue if geom.isMultipart(): points = geom.asMultiPoint() else: points = [geom.asPoint()] for n, point in enumerate(points): x = point.x() y = point.y() pts.append((x, y)) ptNdx += 1 ptDict[ptNdx] = (inFeat.id(), n) feedback.setProgress(int(current * total)) if len(pts) < 3: raise GeoAlgorithmExecutionException( self.tr('Input file should contain at least 3 points. Choose ' 'another file and try again.')) uniqueSet = set(item for item in pts) ids = [pts.index(item) for item in uniqueSet] sl = voronoi.SiteList([voronoi.Site(*i) for i in uniqueSet]) c.triangulate = True voronoi.voronoi(sl, c) triangles = c.triangles feat = QgsFeature() total = 100.0 / len(triangles) for current, triangle in enumerate(triangles): indices = list(triangle) indices.append(indices[0]) polygon = [] attrs = [] step = 0 for index in indices: fid, n = ptDict[ids[index]] request = QgsFeatureRequest().setFilterFid(fid) inFeat = next(layer.getFeatures(request)) geom = QgsGeometry(inFeat.geometry()) if geom.isMultipart(): point = QgsPoint(geom.asMultiPoint()[n]) else: point = QgsPoint(geom.asPoint()) polygon.append(point) if step <= 3: attrs.append(ids[index]) step += 1 feat.setAttributes(attrs) geometry = QgsGeometry().fromPolygon([polygon]) feat.setGeometry(geometry) writer.addFeature(feat) feedback.setProgress(int(current * total)) del writer