def key_comparator(k1, k2): """A comparator for Datastore keys. Comparison is only valid for keys in the same partition. The comparison here is between the list of paths for each key. """ if k1.partition_id != k2.partition_id: raise ValueError('Cannot compare keys with different partition ids.') k2_iter = iter(k2.path) for k1_path in k1.path: k2_path = next(k2_iter, None) if not k2_path: return 1 result = compare_path(k1_path, k2_path) if result != 0: return result k2_path = next(k2_iter, None) if k2_path: return -1 return 0
def __next__(self): """ Return the next Statement in the answer, if there is one. Otherwise raise StopIteration exception. :return: The next statement. :raises StopIteration: If there are no more statements. """ if self.nonDuplicateSet is not None: try: savedNonDuplicateSet = self.nonDuplicateSet self.nonDuplicateSet = None while (True): stmt = next(self) if not stmt in savedNonDuplicateSet: savedNonDuplicateSet.add(stmt) return stmt finally: self.nonDuplicateSet = savedNonDuplicateSet # elif self.limit and self.cursor >= self.limit: # raise StopIteration elif self.cursor < len(self.string_tuples): stringTuple = self.string_tuples[self.cursor] if self.triple_ids: stringTuple = RepositoryResult.normalize_quint(stringTuple) self.cursor += 1 if self.subjectFilter and not stringTuple[0] == self.subjectFilter: return next(self) return self._createStatement(stringTuple); else: raise StopIteration
def test_non_iterator(self): """ The default behaviour of next(o) for a newobject o should be to raise a TypeError, as with the corresponding builtin object. """ o = object() with self.assertRaises(TypeError): next(o)
def test_bond_sample_state_raises_more_than_two_spanning_sides( percolation_graph ): if percolation_graph['spanning_cluster']: percolation_graph['spanning_sides'] = [0, 1, 2] with pytest.raises(ValueError): next(percolate.hpc.bond_sample_states( seed=0, **percolation_graph ))
def lex_label(self): self.eat_spaces() char = self.streamer.peek() if char in ('"', "'"): next(self.streamer) # throw away opening quote self._match_delimited(char) else: despacer = {' ': '_'} self._match_run(str.isalnum, accepted_chars='-_|.', denied_chars=':,;', replacements=despacer) label = self.token_buffer.decode() if label == '': label = None self.emit(Token(self.tokens.LABEL, label)) return self.lex_length
def test_bond_microcanonical_statistics(percolation_graph, seed): result = percolate.hpc.bond_microcanonical_statistics( seed=seed, **percolation_graph ) sample_states = percolate.hpc.bond_sample_states( seed=seed, **percolation_graph ) for n in range(percolation_graph['num_edges'] + 1): result_row = result[n] sample_state = next(sample_states) def _assert_equal(key): assert result_row[key] == sample_state[key][0] assert result_row['n'] == n assert sample_state['n'][0] == n if n > 0: # first edge is undefined! _assert_equal('edge') if percolation_graph['spanning_cluster']: _assert_equal('has_spanning_cluster') _assert_equal('max_cluster_size') np.testing.assert_array_equal( result_row['moments'], sample_state['moments'][0], )
def writeUniqueResults(clustered_dupes, input_file, output_file): # Write our original data back out to a CSV with a new column called # 'Cluster ID' which indicates which records refer to each other. logging.info('saving unique results to: %s' % output_file) cluster_membership = {} for (cluster_id, cluster) in enumerate(clustered_dupes): for record_id in cluster: cluster_membership[record_id] = cluster_id unique_record_id = cluster_id + 1 writer = csv.writer(output_file) reader = csv.reader(StringIO(input_file)) heading_row = next(reader) heading_row.insert(0, u'Cluster ID') writer.writerow(heading_row) seen_clusters = set() for row_id, row in enumerate(reader): if row_id in cluster_membership: cluster_id = cluster_membership[row_id] if cluster_id not in seen_clusters: row.insert(0, cluster_id) writer.writerow(row) seen_clusters.add(cluster_id) else: cluster_id = unique_record_id unique_record_id += 1 row.insert(0, cluster_id) writer.writerow(row)
def save(self, handle, name=None): """ Save the *k*-mer counts to a file. :arg h5py.File handle: Open writeable *k*-mer profile file handle. :arg name: Profile name in the file. If not provided, the current profile name is used, or the first available number from 1 consecutively if the profile has no name. :type name: str :return: Profile name in the file. :rtype: str """ if name and ('/' in name or '.' in name): raise ValueError('Profile name may not contain / or . characters.') name = name or self.name or next(str(n) for n in itertools.count(1) if str(n) not in handle['profiles']) profile = handle.create_dataset('profiles/' + name, data=self.counts, dtype='int64', compression='gzip') profile.attrs['length'] = self.length profile.attrs['total'] = self.total profile.attrs['non_zero'] = self.non_zero profile.attrs['mean'] = self.mean profile.attrs['median'] = self.median profile.attrs['std'] = self.std handle.flush() return name
def test_implements_py2_iterator(self): class Upper(object): def __init__(self, iterable): self._iter = iter(iterable) def __next__(self): # note the Py3 interface return next(self._iter).upper() def __iter__(self): return self self.assertEqual(list(Upper('hello')), list('HELLO')) # Try combining it with the next() function: class MyIter(object): def __next__(self): return 'Next!' def __iter__(self): return self itr = MyIter() self.assertEqual(next(itr), 'Next!') itr2 = MyIter() for i, item in enumerate(itr2): if i >= 10: break self.assertEqual(item, 'Next!')
def demonstrate(device_name): """ Apply function 'static_route_create' to the specified device. Choose a destination that does not already exist. Choose a next-hop on the same sub-network as an existing interface. If the next-hop is unknown then use any valid ip-address. """ print('Select a destination network for which a static route does not exist.') destination_network_iterator = destination_network_generator() while True: destination_network = next(destination_network_iterator) print('static_route_exists(%s, %s)' % (device_name, destination_network)) exists = static_route_exists(device_name, destination_network) print(exists) print() if not exists: break print('Determine which interface is on the management plane (to avoid it).') print('device_control(%s)' % device_name) device_mgmt = device_control(device_name) print_table(device_mgmt) print() print('Determine ip-address and network-mask of every interface.') print('interface_configuration_tuple(%s)' % device_name) interface_config_list = interface_configuration_tuple(device_name) print_table(interface_config_list) print() for interface_config in interface_config_list: if interface_config.address is None: # Skip network interface with unassigned IP address. continue if interface_config.address == device_mgmt.address: # Do not configure static routes on the 'management plane'. continue if 'Loopback' in interface_config.name: # Do not configure static routes on the 'control plane'. continue print('Determine next-hop for a network interface.') interface_network = interface_config.ip_network next_hop_address = match(device_name, interface_network) if next_hop_address is None: print('Next-hop for %s/%s %s/%s is outside the known topology.' % (device_name, interface_config.name, interface_config.address, interface_config.netmask)) next_hop_address = interface_network.network_address if next_hop_address == interface_config.ip_interface: next_hop_address += 1 print('Assume that next-hop for %s/%s %s/%s is %s.' % (device_name, interface_config.name, interface_config.address, interface_config.netmask, next_hop_address)) else: print('Next-hop for %s/%s %s/%s is %s.' % (device_name, interface_config.name, interface_config.address, interface_config.netmask, next_hop_address)) print() print('static_route_create(%s, %s, %s)' % (device_name, destination_network, next_hop_address)) static_route_create(device_name, destination_network, next_hop_address) return True return False
def cleanpy(dirname, recursive=True): walk_it = os.walk(dirname) if not recursive: walk_it = (next(walk_it)) for dirpath, dirnames, filenames in walk_it: _cleanpy2(dirpath, filenames) _cleanpy3(dirpath, dirnames)
def _get_label(self, tokens): """ Get the node data attributes 'label' and 'length'. Assumes these will be the next tokens in the stream. Throws ParseError if they are not. """ label = next(self.lexer) if label.typ not in (tokens.LABEL, tokens.SUPPORT): raise ParseError( 'Expected a label or a support value, found {0}'.format( label)) length = next(self.lexer) if length.typ != tokens.LENGTH: raise ParseError('Expected a length, found {0}'.format( length)) return (label.val if label.typ == tokens.LABEL else None)
def __setSchemaCombo(self, uriDb): """ To fill the schema combo list :param uriDb: selected database uri """ connector = DBConnector(uriDb, self.__iface) db = connector.setConnection() if db: Signal.safelyDisconnect(self.__schemaCombo.currentIndexChanged, self.__schemaComboChanged) self.__resetCombo(self.__schemaCombo) self.__schemaCombo.addItem("") self.__schemas = [] query = db.exec_("""SELECT DISTINCT table_schema FROM information_schema.tables WHERE table_schema NOT IN ('pg_catalog', 'information_schema', 'topology') AND table_type = 'BASE TABLE' AND table_name NOT IN (SELECT f_table_name FROM geometry_columns)""") if query.lastError().isValid(): self.__iface.messageBar().pushMessage(query.lastError().text(), level=QgsMessageBar.CRITICAL, duration=0) else: while next(query): self.__schemas.append(query.value(0)) db.close() for schema in self.__schemas: self.__schemaCombo.addItem(schema) self.__schemaCombo.currentIndexChanged.connect(self.__schemaComboChanged) if self.__schemaDb is not None: if self.__schemaDb in self.__schemas: self.__schemaCombo.setCurrentIndex(self.__schemas.index(self.__schemaDb) + 1)
def __setTableCombo(self, uriDb, schema): """ To fill the table combo list :param uriDb: selected database uri :param schema: selected database schema """ connector = DBConnector(uriDb, self.__iface) db = connector.setConnection() if db: Signal.safelyDisconnect(self.__tableCombo.currentIndexChanged, self.__tableComboChanged) self.__resetCombo(self.__tableCombo) self.__tableCombo.addItem("") self.__tables = [] query = db.exec_("""SELECT table_name FROM information_schema.tables WHERE table_schema = '""" + schema + """' ORDER BY table_name""") if query.lastError().isValid(): self.__iface.messageBar().pushMessage(query.lastError().text(), level=QgsMessageBar.CRITICAL, duration=0) else: while next(query): self.__tables.append(query.value(0)) db.close() for table in self.__tables: if self.__tableCombo.findText(table) == -1: self.__tableCombo.addItem(table) self.__tableCombo.currentIndexChanged.connect(self.__tableComboChanged) if self.__configTable is not None: if self.__configTable in self.__tables: self.__tableCombo.setCurrentIndex(self.__tables.index(self.__configTable) + 1)
def _addTaxRangeR(self, node, noUpwardLevels=False): """recursive method to add TaxRange property tags.""" if self.parser.is_ortholog_group(node) or self.parser.is_paralog_group(node) or OrthoXMLQuery.is_geneRef_node(node): species_covered = self.parser.get_species_below_node(node) current_level = self.tax.mrca(species_covered) try: # find the closest ancestral orthogroup that has a TaxRange property parent_orthogroup_generator = (n for n in node.iterancestors('{{{}}}orthologGroup'.format(OrthoXMLQuery.ns['ns0'])) if n[0].tag == '{{{}}}property'.format(OrthoXMLQuery.ns['ns0'])) parent_orthogroup = next(parent_orthogroup_generator) except: # couldn't find a parent with a TaxRange property; no extra annotation possible parent_orthogroup = None if parent_orthogroup is not None: parent_levels = {z.get('value') for z in OrthoXMLQuery.getTaxRangeNodes(parent_orthogroup, False)} most_recent_parent_level = self.tax.mostSpecific(parent_levels) # Ortholog Node - append missing tax range(s) as property tags under the current node if self.parser.is_ortholog_group(node): for level in self.tax.iterParents(current_level, most_recent_parent_level): node.append(self._createTaxRangeTag(level)) # Paralog Node - insert ortholog node between self and parent; add missing tax range(s) to new parent elif self.parser.is_paralog_group(node): if self.tax.levels_between(most_recent_parent_level, current_level) > 1: self._insertOG(node.getparent(), node, current_level, most_recent_parent_level, include_self=False) # GeneRef Node - insert ortholog node between self and parent; add all tax range(s) to new parent else: self._insertOG(node.getparent(), node, current_level, most_recent_parent_level, include_self=True) return for child in node: self._addTaxRangeR(child, noUpwardLevels)
def from_file_old_format(cls, handle, name=None): """ Load the *k*-mer profile from a file in the old plaintext format. :arg handle: Open readable *k*-mer profile file handle (old format). :type handle: file-like object :arg str name: Profile name. :return: A *k*-mer profile. :rtype: Profile """ # Ignore lines with length, total, nonzero. for _ in range(3): next(handle) counts = np.loadtxt(handle, dtype='int64') return cls(counts, name=name)
def get_question(self, code): if self._lazy: # Trigger full fetch self.name candidates = filter(lambda c: c.code == code, self.questions) if not candidates: return None return next(candidates)
def deserialize(string): def posInteger(chars): result = shift = 0 # Set value to get into the loop the first time value = 0x80 while value & 0x80: value = next(chars) result += ((value & 0x7f) << shift) shift += 7 return result if isinstance(string, old_str): string = bytes(string) chars = iter(string) value = next(chars) if value == SerialConstants.SO_BYTEVECTOR: length = posInteger(chars) import array return array.array(b'b', [ord(next(chars)) for i in range(length)]) if (value == SerialConstants.SO_VECTOR or value == SerialConstants.SO_LIST): length = posInteger(chars) return [deserialize(chars) for i in range(length)] if value == SerialConstants.SO_STRING: length = posInteger(chars) return unicode(ibytes(islice(chars, 0, length)), 'utf-8') if value == SerialConstants.SO_POS_INTEGER: return posInteger(chars) if value == SerialConstants.SO_NEG_INTEGER: return - posInteger(chars) if value == SerialConstants.SO_NULL: return None if value == SerialConstants.SO_END_OF_ITEMS: return None raise ValueError("bad code found by deserializer: %d" % value)
def __populate_text_field(self, herd, field_name, xml_name=''): if not xml_name: xml_name = field_name try: element = next(herd.iter(xml_name)) text = gettext(element) except: raise IOError("Couldn't find '%s' label in xml" % xml_name) self.population[-1][field_name] = text
def find_times(fold=None): """ Find time folders in given or current folder Returns sorted list of times as strings """ search_folder = (fold if fold else os.getcwd()) cur_dir = os.walk(search_folder) root, dirs, files = next(cur_dir) times = [time for time in dirs if is_time(time) is not False] times.sort() return times
def lex_subtree_start(self): self.eat_spaces() char = self.streamer.peek() if char == '(': self.emit(Token(self.tokens.SUBTREE, next(self.streamer))) return self.lex_subtree_start else: self.emit(Token(self.tokens.LEAF, None)) return self.lex_label
def posInteger(chars): result = shift = 0 # Set value to get into the loop the first time value = 0x80 while value & 0x80: value = next(chars) result += ((value & 0x7f) << shift) shift += 7 return result
def __next__(self): while True: try: return self._klass(self.api, next(self._iter)) except StopIteration: next_url = self.jsondata.get("next", False) if not next_url: raise self.jsondata = self.api.connection.make_get(next_url) self._get_iter() raise StopIteration
def _match(self, predicate, accepted_chars='', denied_chars='', replacements=None): """ Checks next character in stream. If predicate returns True, or char is in `accepted_chars`, advances the stream and returns 1. Else, or if the char is in `denied_chars`, doesn't advance the stream and returns 0. Replacements is an optional dictionary that can be used to replace the streamed character with an alternative (e.g. replace spaces with underscores). """ replacements = (replacements or {}) char = self.streamer.peek() char = replacements.get(char, char) if predicate(char) or char in accepted_chars: if len(char) == 1: self.buffer(char) next(self.streamer) # advance stream return 1 elif char in denied_chars: return 0 else: return 0
def demonstrate(device_name): """ Apply function 'static_route_exists' to the specified device for each destination in the list. """ destination_network_iterator = destination_network_generator() while True: destination_network = next(destination_network_iterator) print('static_route_exists(%s, %s)' % (device_name, destination_network)) exists = static_route_exists(device_name, destination_network) print(exists) if not exists: return True else: print()
def read_boundary_names(fn): """ Todo use iterator method to avoid reading complete file """ with open(fn, encoding="utf-8") as f: boundary_names = [] lines = reversed(f.readlines()) for line in lines: if "{" in line: follower = next(lines) if "boundaryField" in follower: return boundary_names try: boundary_names.append(follower) except: pass else: pass
def _get_datafiles_from_dir(path=False, fn_filter=False): """ Return file names of Foam files from cwd if no path is specified explicitly. If no filter list is given the complete list of files will be returned else only files matching that list """ path = (path if path else os.getcwd() + "/") path = (path + "/" if not path.endswith("/") else path) cur_dir = os.walk(path) root, dirs, files = next(cur_dir) if fn_filter: l = [path + f for f in files if f in fn_filter] else: l = [path + f for f in files if not f.startswith('.')] l.sort() return l
def max_elems(iterable, key=None): """Find the elements in 'iterable' corresponding to the maximum values w.r.t. 'key'.""" iterator = iter(iterable) try: elem = next(iterator) except StopIteration: raise ValueError("argument iterable must be non-empty") max_elems = [elem] max_key = elem if key is None else key(elem) for elem in iterator: curr_key = elem if key is None else key(elem) if curr_key > max_key: max_elems = [elem] max_key = curr_key elif curr_key == max_key: max_elems.append(elem) return max_elems
def _render_node(self, context, cl, node): bits = [] context.push() # Render children to add to parent later for child in node.get_children(): bits.append(self._render_node(context, cl, child)) columns = self._get_column_repr(cl, node) # list(tuple(name, html), ..) first_real_column = next(col for col in columns if col[0] != 'action_checkbox') context['columns'] = columns context['other_columns'] = [col for col in columns if col[0] not in ('action_checkbox', first_real_column[0])] context['first_column'] = first_real_column[1] context['named_columns'] = dict(columns) context['node'] = node context['change_url'] = cl.url_for_result(node) context['children'] = mark_safe(u''.join(bits)) # Render rendered = self.template_nodes.render(context) context.pop() return rendered
def __checkIfExist(self): """ To check if the data we want to import is already in the table """ if self.__iter < len(self.__data): data = self.__data[self.__iter] # check if already in table query = self.__db.exec_( """SELECT ST_AsText(geometry3d) FROM %s.%s WHERE st_dwithin('%s', geometry3d, 0.03)""" % (data['schema_table'], data['name_table'],data['geom'])) if query.lastError().isValid(): self.__iface.messageBar().pushMessage( query.lastError().text(), level=QgsMessageBar.CRITICAL, duration=0) else: in_base = False point = None while next(query): point = query.value(0) in_base = True if in_base: self.__data[self.__iter]['point'] = point self.__confDlg = ImportConfirmDialog() self.__confDlg.setMessage( QCoreApplication.translate("VDLTools", "There is already a ") + point + QCoreApplication.translate("VDLTools", " in table ") + data['schema_table'] + """.""" + data['name_table'] + ".\n" + QCoreApplication.translate("VDLTools", "Would you like to add it anyway ? ")) self.__confDlg.rejected.connect(self.__cancelAndNext) self.__confDlg.accepted.connect(self.__confirmAndNext) self.__confDlg.okButton().clicked.connect(self.__onConfirmOk) self.__confDlg.cancelButton().clicked.connect(self.__onConfirmCancel) self.__confDlg.show() else: self.__confirmAndNext() else: self.__insert()
def get_server_resources(return_presence=False, return_server=False, return_info=False, **kwargs): if not return_presence and not return_info: logger.info("Tautulli PlexTV :: Requesting resources for server...") server = { 'pms_name': plexpy.CONFIG.PMS_NAME, 'pms_version': plexpy.CONFIG.PMS_VERSION, 'pms_platform': plexpy.CONFIG.PMS_PLATFORM, 'pms_ip': plexpy.CONFIG.PMS_IP, 'pms_port': plexpy.CONFIG.PMS_PORT, 'pms_ssl': plexpy.CONFIG.PMS_SSL, 'pms_is_remote': plexpy.CONFIG.PMS_IS_REMOTE, 'pms_is_cloud': plexpy.CONFIG.PMS_IS_CLOUD, 'pms_url': plexpy.CONFIG.PMS_URL, 'pms_url_manual': plexpy.CONFIG.PMS_URL_MANUAL, 'pms_identifier': plexpy.CONFIG.PMS_IDENTIFIER, 'pms_plexpass': plexpy.CONFIG.PMS_PLEXPASS } if return_info: return server if kwargs: server.update(kwargs) for k in [ 'pms_ssl', 'pms_is_remote', 'pms_is_cloud', 'pms_url_manual' ]: server[k] = int(server[k]) if server['pms_url_manual'] and server['pms_ssl'] or server['pms_is_cloud']: scheme = 'https' else: scheme = 'http' fallback_url = '{scheme}://{hostname}:{port}'.format( scheme=scheme, hostname=server['pms_ip'], port=server['pms_port']) plex_tv = PlexTV() result = plex_tv.get_server_connections( pms_identifier=server['pms_identifier'], pms_ip=server['pms_ip'], pms_port=server['pms_port'], include_https=server['pms_ssl']) if result: connections = result.pop('connections', []) server.update(result) presence = server.pop('pms_presence', 0) else: connections = [] presence = 0 if return_presence: return presence plexpass = plex_tv.get_plexpass_status() server['pms_plexpass'] = int(plexpass) # Only need to retrieve PMS_URL if using SSL if not server['pms_url_manual'] and server['pms_ssl']: if connections: if server['pms_is_remote']: # Get all remote connections conns = [ c for c in connections if c['local'] == '0' and ('plex.direct' in c['uri'] or 'plex.service' in c['uri']) ] else: # Get all local connections conns = [ c for c in connections if c['local'] == '1' and ('plex.direct' in c['uri'] or 'plex.service' in c['uri']) ] if conns: # Get connection with matching address, otherwise return first connection conn = next( (c for c in conns if c['address'] == server['pms_ip'] and c['port'] == str(server['pms_port'])), conns[0]) server['pms_url'] = conn['uri'] logger.info("Tautulli PlexTV :: Server URL retrieved.") # get_server_urls() failed or PMS_URL not found, fallback url doesn't use SSL if not server['pms_url']: server['pms_url'] = fallback_url logger.warn( "Tautulli PlexTV :: Unable to retrieve server URLs. Using user-defined value without SSL." ) # Not using SSL, remote has no effect else: server['pms_url'] = fallback_url logger.info("Tautulli PlexTV :: Using user-defined URL.") if return_server: return server logger.info( "Tautulli PlexTV :: Selected server: %s (%s) (%s - Version %s)", server['pms_name'], server['pms_url'], server['pms_platform'], server['pms_version']) plexpy.CONFIG.process_kwargs(server) plexpy.CONFIG.write()
def get_synced_items(self, machine_id=None, client_id_filter=None, user_id_filter=None, rating_key_filter=None, sync_id_filter=None): if not machine_id: machine_id = plexpy.CONFIG.PMS_IDENTIFIER if isinstance(rating_key_filter, list): rating_key_filter = [str(k) for k in rating_key_filter] elif rating_key_filter: rating_key_filter = [str(rating_key_filter)] if isinstance(user_id_filter, list): user_id_filter = [str(k) for k in user_id_filter] elif user_id_filter: user_id_filter = [str(user_id_filter)] sync_list = self.get_plextv_sync_lists(machine_id, output_format='xml') user_data = users.Users() synced_items = [] try: xml_head = sync_list.getElementsByTagName('SyncList') except Exception as e: logger.warn( "Tautulli PlexTV :: Unable to parse XML for get_synced_items: %s." % e) return {} for a in xml_head: client_id = helpers.get_xml_attr(a, 'clientIdentifier') # Filter by client_id if client_id_filter and str(client_id_filter) != client_id: continue sync_list_id = helpers.get_xml_attr(a, 'id') sync_device = a.getElementsByTagName('Device') for device in sync_device: device_user_id = helpers.get_xml_attr(device, 'userID') try: device_username = user_data.get_details( user_id=device_user_id)['username'] device_friendly_name = user_data.get_details( user_id=device_user_id)['friendly_name'] except: device_username = '' device_friendly_name = '' device_name = helpers.get_xml_attr(device, 'name') device_product = helpers.get_xml_attr(device, 'product') device_product_version = helpers.get_xml_attr( device, 'productVersion') device_platform = helpers.get_xml_attr(device, 'platform') device_platform_version = helpers.get_xml_attr( device, 'platformVersion') device_type = helpers.get_xml_attr(device, 'device') device_model = helpers.get_xml_attr(device, 'model') device_last_seen = helpers.get_xml_attr(device, 'lastSeenAt') # Filter by user_id if user_id_filter and device_user_id not in user_id_filter: continue for synced in a.getElementsByTagName('SyncItems'): sync_item = synced.getElementsByTagName('SyncItem') for item in sync_item: for location in item.getElementsByTagName('Location'): clean_uri = helpers.get_xml_attr(location, 'uri').split('%2F') rating_key = next((clean_uri[(idx + 1) % len(clean_uri)] for idx, item in enumerate(clean_uri) if item == 'metadata'), None) # Filter by rating_key if rating_key_filter and rating_key not in rating_key_filter: continue sync_id = helpers.get_xml_attr(item, 'id') # Filter by sync_id if sync_id_filter and str(sync_id_filter) != sync_id: continue sync_version = helpers.get_xml_attr(item, 'version') sync_root_title = helpers.get_xml_attr(item, 'rootTitle') sync_title = helpers.get_xml_attr(item, 'title') sync_metadata_type = helpers.get_xml_attr( item, 'metadataType') sync_content_type = helpers.get_xml_attr( item, 'contentType') for status in item.getElementsByTagName('Status'): status_failure_code = helpers.get_xml_attr( status, 'failureCode') status_failure = helpers.get_xml_attr( status, 'failure') status_state = helpers.get_xml_attr(status, 'state') status_item_count = helpers.get_xml_attr( status, 'itemsCount') status_item_complete_count = helpers.get_xml_attr( status, 'itemsCompleteCount') status_item_downloaded_count = helpers.get_xml_attr( status, 'itemsDownloadedCount') status_item_ready_count = helpers.get_xml_attr( status, 'itemsReadyCount') status_item_successful_count = helpers.get_xml_attr( status, 'itemsSuccessfulCount') status_total_size = helpers.get_xml_attr( status, 'totalSize') status_item_download_percent_complete = helpers.get_percent( status_item_downloaded_count, status_item_count) for settings in item.getElementsByTagName('MediaSettings'): settings_video_bitrate = helpers.get_xml_attr( settings, 'maxVideoBitrate') settings_video_quality = helpers.get_xml_attr( settings, 'videoQuality') settings_video_resolution = helpers.get_xml_attr( settings, 'videoResolution') settings_audio_boost = helpers.get_xml_attr( settings, 'audioBoost') settings_audio_bitrate = helpers.get_xml_attr( settings, 'musicBitrate') settings_photo_quality = helpers.get_xml_attr( settings, 'photoQuality') settings_photo_resolution = helpers.get_xml_attr( settings, 'photoResolution') sync_details = { "device_name": device_name, "platform": device_platform, "user_id": device_user_id, "user": device_friendly_name, "username": device_username, "root_title": sync_root_title, "sync_title": sync_title, "metadata_type": sync_metadata_type, "content_type": sync_content_type, "rating_key": rating_key, "state": status_state, "item_count": status_item_count, "item_complete_count": status_item_complete_count, "item_downloaded_count": status_item_downloaded_count, "item_downloaded_percent_complete": status_item_download_percent_complete, "video_bitrate": settings_video_bitrate, "audio_bitrate": settings_audio_bitrate, "photo_quality": settings_photo_quality, "video_quality": settings_video_quality, "total_size": status_total_size, "failure": status_failure, "client_id": client_id, "sync_id": sync_id } synced_items.append(sync_details) return session.filter_session_info(synced_items, filter_key='user_id')
def realseq(): return next(seqnum_counter), str(os.randint(1, 100000))
def test_microcanonical_averages_noninteger_runs(grid_3x3_graph): with pytest.raises(ValueError): next(percolate.microcanonical_averages(grid_3x3_graph, runs='many'))
def constructColName2IndexFromHeader(self): """ """ self.header = next(self) self.col_name2index = utils.getColName2IndexFromHeader(self.header) return self.col_name2index
def check_github(scheduler=False, notify=False, use_cache=False): plexpy.COMMITS_BEHIND = 0 if plexpy.CONFIG.GIT_TOKEN: headers = {'Authorization': 'token {}'.format(plexpy.CONFIG.GIT_TOKEN)} else: headers = {} version = github_cache('version', use_cache=use_cache) if not version: # Get the latest version available from github logger.info('Retrieving latest version information from GitHub') url = 'https://api.github.com/repos/%s/%s/commits/%s' % ( plexpy.CONFIG.GIT_USER, plexpy.CONFIG.GIT_REPO, plexpy.CONFIG.GIT_BRANCH) version = request.request_json(url, headers=headers, timeout=20, validator=lambda x: type(x) == dict) github_cache('version', github_data=version) if version is None: logger.warn( 'Could not get the latest version from GitHub. Are you running a local development version?' ) return plexpy.CURRENT_VERSION plexpy.LATEST_VERSION = version['sha'] logger.debug("Latest version is %s", plexpy.LATEST_VERSION) # See how many commits behind we are if not plexpy.CURRENT_VERSION: logger.info( 'You are running an unknown version of Tautulli. Run the updater to identify your version' ) return plexpy.LATEST_VERSION if plexpy.LATEST_VERSION == plexpy.CURRENT_VERSION: logger.info('Tautulli is up to date') return plexpy.LATEST_VERSION commits = github_cache('commits', use_cache=use_cache) if not commits: logger.info( 'Comparing currently installed version with latest GitHub version') # Need to compare CURRENT << LATEST to get a list of commits url = 'https://api.github.com/repos/%s/%s/compare/%s...%s' % ( plexpy.CONFIG.GIT_USER, plexpy.CONFIG.GIT_REPO, plexpy.CURRENT_VERSION, plexpy.LATEST_VERSION) commits = request.request_json(url, headers=headers, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) github_cache('commits', github_data=commits) if commits is None: logger.warn('Could not get commits behind from GitHub.') return plexpy.LATEST_VERSION try: ahead_by = int(commits['ahead_by']) logger.debug("In total, %d commits behind", ahead_by) # Do not count [skip ci] commits for Docker or Snap on the nightly branch if (plexpy.DOCKER or plexpy.SNAP) and plexpy.CONFIG.GIT_BRANCH == 'nightly': for commit in reversed(commits['commits']): if '[skip ci]' not in commit['commit']['message']: plexpy.LATEST_VERSION = commit['sha'] break ahead_by -= 1 install = 'Docker container' if plexpy.DOCKER else 'Snap package' logger.debug("%s %d commits behind", install, ahead_by) plexpy.COMMITS_BEHIND = ahead_by except KeyError: logger.info( 'Cannot compare versions. Are you running a local development version?' ) plexpy.COMMITS_BEHIND = 0 if plexpy.COMMITS_BEHIND > 0: logger.info('New version is available. You are %s commits behind' % plexpy.COMMITS_BEHIND) releases = github_cache('releases', use_cache=use_cache) if not releases: url = 'https://api.github.com/repos/%s/%s/releases' % ( plexpy.CONFIG.GIT_USER, plexpy.CONFIG.GIT_REPO) releases = request.request_json( url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) github_cache('releases', github_data=releases) if releases is None: logger.warn('Could not get releases from GitHub.') return plexpy.LATEST_VERSION if plexpy.CONFIG.GIT_BRANCH == 'master': release = next((r for r in releases if not r['prerelease']), releases[0]) elif plexpy.CONFIG.GIT_BRANCH == 'beta': release = next( (r for r in releases if not r['tag_name'].endswith('-nightly')), releases[0]) elif plexpy.CONFIG.GIT_BRANCH == 'nightly': release = next((r for r in releases), releases[0]) else: release = releases[0] plexpy.LATEST_RELEASE = release['tag_name'] if plexpy.CONFIG.GIT_BRANCH in ( 'master', 'beta' ) and release['target_commitish'] == plexpy.CURRENT_VERSION: logger.info('Tautulli is up to date') return plexpy.CURRENT_VERSION if notify: plexpy.NOTIFY_QUEUE.put({ 'notify_action': 'on_plexpyupdate', 'plexpy_download_info': release, 'plexpy_update_commit': plexpy.LATEST_VERSION, 'plexpy_update_behind': plexpy.COMMITS_BEHIND }) if plexpy.PYTHON2: logger.warn( 'Tautulli is running using Python 2. Unable to run automatic update.' ) elif scheduler and plexpy.CONFIG.PLEXPY_AUTO_UPDATE and \ not plexpy.DOCKER and not plexpy.SNAP and not plexpy.FROZEN: logger.info('Running automatic update.') plexpy.shutdown(restart=True, update=True) elif plexpy.COMMITS_BEHIND == 0: logger.info('Tautulli is up to date') return plexpy.LATEST_VERSION
from __future__ import print_function from future.builtins import object from future.builtins import next from future import standard_library import configparser # Py3-style import standard_library.install_aliases() class Upper(object): def __init__(self, iterable): self._iter = iter(iterable) def __next__(self): # Py3-style iterator interface return next(self._iter).upper() def __iter__(self): return self itr = Upper('hello') print(next(itr), end=' ') # Py3-style print function for letter in itr: print(letter, end=' ')
def test_sample_state_one_sided_auxiliary_nodes(empty_graph): empty_graph.add_node(1, span=0) with pytest.raises(ValueError): next(percolate.sample_states(empty_graph, spanning_cluster=True))
def __next__(self): # note the Py3 interface return next(self._iter).upper()
def test_sample_state_not_implemented_model(empty_graph): with pytest.raises(ValueError): next(percolate.sample_states(empty_graph, model='site'))
def test_sample_state_no_auxiliary_nodes(empty_graph): with pytest.raises(ValueError): next(percolate.sample_states(empty_graph, spanning_cluster=True))
def test_microcanonical_averages_zero_alpha(grid_3x3_graph): with pytest.raises(ValueError): next(percolate.microcanonical_averages(grid_3x3_graph, alpha=0.0))
def test_microcanonical_averages_nonfloat_alpha(grid_3x3_graph): with pytest.raises(ValueError): next(percolate.microcanonical_averages(grid_3x3_graph, alpha='huge'))
def test_microcanonical_averages_nonpositive_runs(grid_3x3_graph): with pytest.raises(ValueError): next(percolate.microcanonical_averages(grid_3x3_graph, runs=0))
def header_encode_lines(self, string, maxlengths): """Header-encode a string by converting it first to bytes. This is similar to `header_encode()` except that the string is fit into maximum line lengths as given by the argument. :param string: A unicode string for the header. It must be possible to encode this string to bytes using the character set's output codec. :param maxlengths: Maximum line length iterator. Each element returned from this iterator will provide the next maximum line length. This parameter is used as an argument to built-in next() and should never be exhausted. The maximum line lengths should not count the RFC 2047 chrome. These line lengths are only a hint; the splitter does the best it can. :return: Lines of encoded strings, each with RFC 2047 chrome. """ # See which encoding we should use. codec = self.output_codec or "us-ascii" header_bytes = _encode(string, codec) encoder_module = self._get_encoder(header_bytes) encoder = partial(encoder_module.header_encode, charset=codec) # Calculate the number of characters that the RFC 2047 chrome will # contribute to each line. charset = self.get_output_charset() extra = len(charset) + RFC2047_CHROME_LEN # Now comes the hard part. We must encode bytes but we can't split on # bytes because some character sets are variable length and each # encoded word must stand on its own. So the problem is you have to # encode to bytes to figure out this word's length, but you must split # on characters. This causes two problems: first, we don't know how # many octets a specific substring of unicode characters will get # encoded to, and second, we don't know how many ASCII characters # those octets will get encoded to. Unless we try it. Which seems # inefficient. In the interest of being correct rather than fast (and # in the hope that there will be few encoded headers in any such # message), brute force it. :( lines = [] current_line = [] maxlen = next(maxlengths) - extra for character in string: current_line.append(character) this_line = EMPTYSTRING.join(current_line) length = encoder_module.header_length(_encode(this_line, charset)) if length > maxlen: # This last character doesn't fit so pop it off. current_line.pop() # Does nothing fit on the first line? if not lines and not current_line: lines.append(None) else: separator = " " if lines else "" joined_line = EMPTYSTRING.join(current_line) header_bytes = _encode(joined_line, codec) lines.append(encoder(header_bytes)) current_line = [character] maxlen = next(maxlengths) - extra joined_line = EMPTYSTRING.join(current_line) header_bytes = _encode(joined_line, codec) lines.append(encoder(header_bytes)) return lines
def advance_i2(self): try: val = next(self.i2) except StopIteration: val = None self.f2 = val
def sample_states(graph, spanning_cluster=True, model='bond', copy_result=True): ''' Generate successive sample states of the percolation model This is a :ref:`generator function <python:tut-generators>` to successively add one edge at a time from the graph to the percolation model. At each iteration, it calculates and returns the cluster statistics. Parameters ---------- graph : networkx.Graph The substrate graph on which percolation is to take place spanning_cluster : bool, optional Whether to detect a spanning cluster or not. Defaults to ``True``. model : str, optional The percolation model (either ``'bond'`` or ``'site'``). Defaults to ``'bond'``. .. note:: Other models than ``'bond'`` are not supported yet. copy_result : bool, optional Whether to return a copy or a reference to the result dictionary. Defaults to ``True``. Yields ------ ret : dict Cluster statistics ret['n'] : int Number of occupied bonds ret['N'] : int Total number of sites ret['M'] : int Total number of bonds ret['has_spanning_cluster'] : bool ``True`` if there is a spanning cluster, ``False`` otherwise. Only exists if `spanning_cluster` argument is set to ``True``. ret['max_cluster_size'] : int Size of the largest cluster (absolute number of sites) ret['moments'] : 1-D :py:class:`numpy.ndarray` of int Array of size ``5``. The ``k``-th entry is the ``k``-th raw moment of the (absolute) cluster size distribution, with ``k`` ranging from ``0`` to ``4``. Raises ------ ValueError If `model` does not equal ``'bond'``. ValueError If `spanning_cluster` is ``True``, but `graph` does not contain any auxiliary nodes to detect spanning clusters. See also -------- microcanonical_averages : Evolves multiple sample states in parallel Notes ----- Iterating through this generator is a single run of the Newman-Ziff algorithm. [2]_ The first iteration yields the trivial state with :math:`n = 0` occupied bonds. Spanning cluster In order to detect a spanning cluster, `graph` needs to contain auxiliary nodes and edges, cf. Reference [2]_, Figure 6. The auxiliary nodes and edges have the ``'span'`` `attribute <http://networkx.github.io/documentation/latest/tutorial/tutorial.html#node-attributes>`_. The value is either ``0`` or ``1``, distinguishing the two sides of the graph to span. Raw moments of the cluster size distribution The :math:`k`-th raw moment of the (absolute) cluster size distribution is :math:`\sum_s' s^k N_s`, where :math:`s` is the cluster size and :math:`N_s` is the number of clusters of size :math:`s`. [3]_ The primed sum :math:`\sum'` signifies that the largest cluster is excluded from the sum. [4]_ References ---------- .. [2] Newman, M. E. J. & Ziff, R. M. Fast monte carlo algorithm for site or bond percolation. Physical Review E 64, 016706+ (2001), `doi:10.1103/physreve.64.016706 <http://dx.doi.org/10.1103/physreve.64.016706>`_. .. [3] Stauffer, D. & Aharony, A. Introduction to Percolation Theory (Taylor & Francis, London, 1994), second edn. .. [4] Binder, K. & Heermann, D. W. Monte Carlo Simulation in Statistical Physics (Springer, Berlin, Heidelberg, 2010), `doi:10.1007/978-3-642-03163-2 <http://dx.doi.org/10.1007/978-3-642-03163-2>`_. ''' if model != 'bond': raise ValueError('Only bond percolation supported.') if spanning_cluster: auxiliary_node_attributes = nx.get_node_attributes(graph, 'span') auxiliary_nodes = auxiliary_node_attributes.keys() if not list(auxiliary_nodes): raise ValueError( 'Spanning cluster is to be detected, but no auxiliary nodes ' 'given.') spanning_sides = list(set(auxiliary_node_attributes.values())) if len(spanning_sides) != 2: raise ValueError( 'Spanning cluster is to be detected, but auxiliary nodes ' 'of less or more than 2 types (sides) given.') auxiliary_edge_attributes = nx.get_edge_attributes(graph, 'span') # get subgraph on which percolation is to take place (strip off the # auxiliary nodes) if spanning_cluster: perc_graph = graph.subgraph([ node for node in graph.nodes_iter() if 'span' not in graph.node[node] ]) else: perc_graph = graph # get a list of edges for easy access in later iterations perc_edges = perc_graph.edges() # number of nodes N num_nodes = nx.number_of_nodes(perc_graph) # number of edges M num_edges = nx.number_of_edges(perc_graph) # initial iteration: no edges added yet (n == 0) ret = dict() ret['n'] = 0 ret['N'] = num_nodes ret['M'] = num_edges ret['max_cluster_size'] = 1 ret['moments'] = np.ones(5) * (num_nodes - 1) if spanning_cluster: ret['has_spanning_cluster'] = False if copy_result: yield copy.deepcopy(ret) else: yield ret # permute edges perm_edges = np.random.permutation(num_edges) # set up disjoint set (union-find) data structure ds = nx.utils.union_find.UnionFind() if spanning_cluster: ds_spanning = nx.utils.union_find.UnionFind() # merge all auxiliary nodes for each side side_roots = dict() for side in spanning_sides: nodes = [ node for (node, node_side) in auxiliary_node_attributes.items() if node_side is side ] ds_spanning.union(*nodes) side_roots[side] = ds_spanning[nodes[0]] for (edge, edge_side) in auxiliary_edge_attributes.items(): ds_spanning.union(side_roots[edge_side], *edge) side_roots = [ ds_spanning[side_root] for side_root in side_roots.values() ] # get first node max_cluster_root = next(perc_graph.nodes_iter()) # loop over all edges (n == 1..M) for n in range(num_edges): ret['n'] = n + 1 # draw new edge from permutation edge_index = perm_edges[n] edge = perc_edges[edge_index] ret['edge'] = edge # find roots and weights roots = [ds[node] for node in edge] weights = [ds.weights[root] for root in roots] if roots[0] is not roots[1]: # not same cluster: union! ds.union(*roots) if spanning_cluster: ds_spanning.union(*roots) ret['has_spanning_cluster'] = ( ds_spanning[side_roots[0]] == ds_spanning[side_roots[1]]) # find new root and weight root = ds[edge[0]] weight = ds.weights[root] # moments and maximum cluster size # deduct the previous sub-maximum clusters from moments for i in [0, 1]: if roots[i] is max_cluster_root: continue ret['moments'] -= weights[i]**np.arange(5) if max_cluster_root in roots: # merged with maximum cluster max_cluster_root = root ret['max_cluster_size'] = weight else: # merged previously sub-maximum clusters if ret['max_cluster_size'] >= weight: # previously largest cluster remains largest cluster # add merged cluster to moments ret['moments'] += weight**np.arange(5) else: # merged cluster overtook previously largest cluster # add previously largest cluster to moments max_cluster_root = root ret['moments'] += ret['max_cluster_size']**np.arange(5) ret['max_cluster_size'] = weight if copy_result: yield copy.deepcopy(ret) else: yield ret
def cluster(self): linker = self.deduper data_1 = self.data_1 if self.data_1 else self._read_data(self.input_file_base) data_2 = self.data_2 if self.data_2 else self._read_data(self.input_file) output_file = self.output_file # threshold1 = linker.threshold(data_1, data_2, recall_weight=2) linked_records = linker.match(data_1, data_2, threshold=0.1) # linked_records = [data for data in linked_records] print(linked_records) # import pdb;pdb.set_trace() print('# duplicate sets', len(linked_records)) # ## Writing Results # Write our original data back out to a CSV with a new column called # 'Cluster ID' which indicates which records refer to each other. # import pdb; pdb.set_trace() cluster_membership = {} cluster_id = None for cluster_id, (cluster, score) in enumerate(linked_records): for record_id in cluster: cluster_membership[record_id] = (cluster_id, score) print(cluster_membership) if cluster_id : unique_id = cluster_id + 1 else : unique_id =0 with open(output_file, 'w') as f: writer = csv.writer(f) header_unwritten = True for fileno, filename in enumerate((self.input_file_base, self.input_file)) : with open(filename) as f_input : reader = csv.reader(f_input) if header_unwritten : heading_row = next(reader) heading_row.insert(0, 'source file') heading_row.insert(0, 'Link Score') heading_row.insert(0, 'Cluster ID') writer.writerow(heading_row) header_unwritten = False else : next(reader) for row_id, row in enumerate(reader): # print(str(filename)+ str(row_id)) # import pdb;pdb.set_trace() cluster_details = cluster_membership.get(str(filename) + str(row[0])) if cluster_details is None : # cluster_id = unique_id # unique_id += 1 # score = None continue else : cluster_id, score = cluster_details row.insert(0, fileno) row.insert(0, score) row.insert(0, cluster_id) writer.writerow(row)
def __next__(self): # Py3-style iterator interface return next(self._iter).upper()
def start(self): """ To start the importation """ if self.ownSettings is None: self.__iface.messageBar().pushMessage(QCoreApplication.translate("VDLTools", "No settings given !!"), level=QgsMessageBar.CRITICAL, duration=0) return if self.ownSettings.importUriDb is None: self.__iface.messageBar().pushMessage(QCoreApplication.translate("VDLTools", "No import db given !!"), level=QgsMessageBar.CRITICAL, duration=0) return if self.ownSettings.importSchemaDb is None: self.__iface.messageBar().pushMessage(QCoreApplication.translate("VDLTools", "No import db schema given !!"), level=QgsMessageBar.CRITICAL, duration=0) return if self.ownSettings.importConfigTable is None: self.__iface.messageBar().pushMessage(QCoreApplication.translate("VDLTools", "No import config table given !!"), level=QgsMessageBar.CRITICAL, duration=0) return self.__configTable = self.ownSettings.importConfigTable self.__schemaDb = self.ownSettings.importSchemaDb self.__connector = DBConnector(self.ownSettings.importUriDb, self.__iface) self.__db = self.__connector.setConnection() if self.__db is not None: query = self.__db.exec_("""SELECT DISTINCT sourcelayer_name FROM %s.%s WHERE sourcelayer_name IS NOT NULL""" % (self.__schemaDb, self.__configTable)) if query.lastError().isValid(): self.__iface.messageBar().pushMessage( query.lastError().text(), level=QgsMessageBar.CRITICAL, duration=0) self.__cancel() else: while next(query): if self.__sourceTable == "": self.__sourceTable = query.value(0) elif self.__sourceTable != query.value(0): self.__iface.messageBar().pushMessage( QCoreApplication.translate("VDLTools", "different sources in config table ?!?"), level=QgsMessageBar.WARNING) for layer in self.__iface.mapCanvas().layers(): if layer is not None and layer.type() == QgsMapLayer.VectorLayer and \ layer.providerType() == "postgres": uri = QgsDataSourceURI(layer.source()) if self.__sourceTable == uri.schema() + "." + uri.table(): self.__selectedFeatures = [] for f in layer.selectedFeatures(): self.__selectedFeatures.append(f.attribute("ID")) break # select jobs query = self.__db.exec_(("""SELECT DISTINCT usr_session_name FROM %s WHERE """ % self.__sourceTable) + """usr_valid = FALSE AND usr_session_name IS NOT NULL""") if query.lastError().isValid(): self.__iface.messageBar().pushMessage( query.lastError().text(), level=QgsMessageBar.CRITICAL, duration=0) self.__cancel() else: jobs = [] while next(query): jobs.append(query.value(0)) if len(jobs) == 0 and (self.__selectedFeatures is None or len(self.__selectedFeatures) == 0): self.__cancel() else: selected = True if self.__selectedFeatures is None or len(self.__selectedFeatures) == 0: selected = False self.__jobsDlg = ImportJobsDialog(jobs, selected) self.__jobsDlg.jobsRadio().clicked.connect(self.__onJobsRadio) if self.__jobsDlg.pointsRadio() is not None: self.__jobsDlg.pointsRadio().clicked.connect(self.__onPointsRadio) self.__jobsDlg.rejected.connect(self.__cancel) self.__jobsDlg.okButton().clicked.connect(self.__onOk) self.__jobsDlg.cancelButton().clicked.connect(self.__onCancel) self.__jobsDlg.show()
def set_newsletter_config(newsletter_id=None, agent_id=None, **kwargs): if str(agent_id).isdigit(): agent_id = int(agent_id) else: logger.error("Tautulli Newsletters :: Unable to set existing newsletter: invalid agent_id %s." % agent_id) return False agent = next((a for a in available_newsletter_agents() if a['id'] == agent_id), None) if not agent: logger.error("Tautulli Newsletters :: Unable to retrieve existing newsletter agent: invalid agent_id %s." % agent_id) return False config_prefix = 'newsletter_config_' email_config_prefix = 'newsletter_email_' newsletter_config = {k[len(config_prefix):]: kwargs.pop(k) for k in list(kwargs.keys()) if k.startswith(config_prefix)} email_config = {k[len(email_config_prefix):]: kwargs.pop(k) for k in list(kwargs.keys()) if k.startswith(email_config_prefix)} for cfg, val in email_config.items(): # Check for a password config keys and a blank password from the HTML form if 'password' in cfg and val == ' ': # Get the previous password so we don't overwrite it with a blank value old_newsletter_config = get_newsletter_config(newsletter_id=newsletter_id) email_config[cfg] = old_newsletter_config['email_config'][cfg] subject = kwargs.pop('subject') body = kwargs.pop('body') message = kwargs.pop('message') agent_class = get_agent_class(agent_id=agent['id'], config=newsletter_config, email_config=email_config, subject=subject, body=body, message=message) keys = {'id': newsletter_id} values = {'agent_id': agent['id'], 'agent_name': agent['name'], 'agent_label': agent['label'], 'id_name': kwargs.get('id_name', ''), 'friendly_name': kwargs.get('friendly_name', ''), 'newsletter_config': json.dumps(agent_class.config), 'email_config': json.dumps(agent_class.email_config), 'subject': agent_class.subject, 'body': agent_class.body, 'message': agent_class.message, 'cron': kwargs.get('cron'), 'active': kwargs.get('active') } db = database.MonitorDatabase() try: db.upsert(table_name='newsletters', key_dict=keys, value_dict=values) logger.info("Tautulli Newsletters :: Updated newsletter agent: %s (newsletter_id %s)." % (agent['label'], newsletter_id)) newsletter_handler.schedule_newsletters(newsletter_id=newsletter_id) blacklist_logger() return True except Exception as e: logger.warn("Tautulli Newsletters :: Unable to update newsletter agent: %s." % e) return False
def __insert(self): """ To insert the data into the tables """ not_added = [] for data in self.__data: if data['add']: destLayer = "" request = """INSERT INTO %s.%s""" % (data['schema_table'], data['name_table']) columns = "(id,geometry3d" values = """(nextval('%s.%s_id_seq'::regclass),'%s'""" \ % (data['schema_table'], data['name_table'], data['geom']) # select import data for insertion query = self.__db.exec_("""SELECT destinationlayer_name,destinationcolumn_name,static_value FROM """ + """"%s.%s WHERE code = '%s' AND static_value IS NOT NULL""" % (self.__schemaDb, self.__configTable, str(data['code']))) if query.lastError().isValid(): self.__iface.messageBar().pushMessage( query.lastError().text(), level=QgsMessageBar.CRITICAL, duration=0) else: while next(query): if destLayer == "": destLayer = query.value(0) elif destLayer != query.value(0): self.__iface.messageBar().pushMessage( QCoreApplication.translate("VDLTools", "different destination layer in config table ?!?"), level=QgsMessageBar.WARNING) columns += "," + query.value(1) values += "," + query.value(2) columns += ")" values += ")" request += """ %s VALUES %s RETURNING id""" % (columns, values) # insert data query2 = self.__db.exec_(request) if query2.lastError().isValid(): self.__iface.messageBar().pushMessage( query2.lastError().text(), level=QgsMessageBar.CRITICAL, duration=0) else: self.__num += 1 query2.first() id_object = query2.value(0) # update source table query3 = self.__db.exec_( ("""UPDATE %s SET usr_valid_date = '%s', usr_valid = TRUE, usr_fk_network_element = %s,""" % (self.__sourceTable, str(datetime.date(datetime.now())), str(id_object))) + (""" usr_fk_table = %s, usr_import_user = '******' WHERE id = %s""" % (str(data['id_table']), self.__db.userName(), str(data['id_survey'])))) if query3.lastError().isValid(): self.__iface.messageBar().pushMessage(query3.lastError().text(), level=QgsMessageBar.CRITICAL, duration=0) else: not_added.append(data) if len(not_added) > 0: self.__measDlg = ImportMeasuresDialog(not_added) self.__measDlg.rejected.connect(self.__validAndNext) self.__measDlg.accepted.connect(self.__deleteAndNext) self.__measDlg.okButton().clicked.connect(self.__onDeleteOk) self.__measDlg.cancelButton().clicked.connect(self.__onDeleteCancel) self.__measDlg.show() else: self.__conclude()
def _addTaxRangeR(self, node, noUpwardLevels=False): """recursive method to add TaxRange property tags.""" if self.parser.is_ortholog_group(node) or self.parser.is_paralog_group(node) \ or OrthoXMLQuery.is_geneRef_node(node): species_covered, nr_genes = self.parser.get_species_below_node( node, return_gene_total_count=True) species_covered = set( self.tax.map_potential_internal_speciesname_to_leaf(s) for s in species_covered) current_level = self.tax.mrca(species_covered) og_tag = '{{{}}}orthologGroup'.format(OrthoXMLQuery.ns['ns0']) if self.parser.is_ortholog_group(node): comp_score = OrthoXMLQuery.getScoreNodes( node, 'CompletenessScore') if len(comp_score) == 0: node.append( self._createCompletnessScoreTag( current_level, species_covered)) node.append(self._createNrMemberGeneTag(nr_genes)) taxrange = OrthoXMLQuery.getTaxRangeNodes(node, False) taxid = OrthoXMLQuery.getTaxidNodes(node, False) if len(taxrange) > 0: # check consistency between current_level and value stored in taxrange if taxrange[0].get('value') != current_level: raise Exception( "Inconsistent TaxRange: {} vs current_level {}". format(taxrange[0].get('value'), current_level)) if len(taxid) > 0: if taxid[0].get('value') != self.taxrange_2_taxid[ current_level]: raise Exception( "Inconsitency between taxids: {} vs {}".format( taxid[0].get('value'), self.taxrange_2_taxid[current_level])) else: try: node.append(self._create_taxid(current_level)) except KeyError: pass else: node.append(self._createTaxRangeTags(current_level)) try: # find the closest ancestral orthogroup if it has a TaxRange property parent_orthogroup = next(node.iterancestors(og_tag)) parent_levels = { z.get('value') for z in OrthoXMLQuery.getTaxRangeNodes( parent_orthogroup, False) } except StopIteration: # couldn't find a parent with a TaxRange property; no extra annotation possible parent_levels = set([]) if len(parent_levels) > 0: most_recent_parent_level = self.tax.mostSpecific(parent_levels) # Ortholog Node - append missing tax range(s) as property tags under the current node if self.parser.is_ortholog_group(node): self._insertOGs_between(node.getparent(), node, current_level, most_recent_parent_level, nr_genes, species_covered, include_self=False) # Paralog Node - insert ortholog node between self and parent; add missing tax range(s) to new parent elif self.parser.is_paralog_group(node): if self.tax.levels_between(most_recent_parent_level, current_level) > 1: self._insertOGs_between(node.getparent(), node, current_level, most_recent_parent_level, nr_genes, species_covered, include_self=False) # GeneRef Node - insert ortholog node between self and parent; add all tax range(s) to new parent else: self._insertOGs_between(node.getparent(), node, current_level, most_recent_parent_level, nr_genes, species_covered, include_self=True) return for child in node: self._addTaxRangeR(child, noUpwardLevels)
cluster_d = [data_d[c] for c in id_set] canonical_rep = dedupe.canonicalize(cluster_d) for record_id, score in zip(id_set, scores): cluster_membership[record_id] = { "cluster id": cluster_id, "canonical representation": canonical_rep, "confidence": score } singleton_id = cluster_id + 1 with open(output_file, 'w') as f_output, open(input_file) as f_input: writer = csv.writer(f_output) reader = csv.reader(f_input) heading_row = next(reader) heading_row.insert(0, 'confidence_score') heading_row.insert(0, 'Cluster ID') canonical_keys = canonical_rep.keys() for key in canonical_keys: heading_row.append('canonical_' + key) writer.writerow(heading_row) for row in reader: row_id = int(row[0]) if row_id in cluster_membership: cluster_id = cluster_membership[row_id]["cluster id"] canonical_rep = cluster_membership[row_id][ "canonical representation"] row.insert(0, cluster_membership[row_id]['confidence'])
def advance_i1(self): try: val = next(self.i1) except StopIteration: val = None self.f1 = val
def eat_spaces(self): while self.streamer.peek().isspace(): next(self.streamer)
def deDuplication(): input_file = 'csv_example_input.csv' output_file = 'csv_example_output.csv' settings_file = 'csv_example_learned_settings3' training_file = 'csv_example_training.json3' def preProcess(column): try: column = column.decode('utf-8') except AttributeError: pass column = unidecode(column) column = re.sub(' +', ' ', column) column = re.sub('\n', ' ', column) column = column.strip().strip('"').strip("'").lower().strip() if not column: column = None return column # Read in the data from CSV file: def readData(filename): data_d = {} with open(filename, encoding="utf-8") as f: reader = csv.DictReader(f) for row in reader: clean_row = [(k, preProcess(v)) for (k, v) in row.items()] row_id = row['id'] data_d[row_id] = dict(clean_row) return data_d print('importing data ...') data_d = readData(input_file) if os.path.exists(settings_file): print('reading from', settings_file) with open(settings_file, 'rb') as f: deduper = dedupe.StaticDedupe(f) else: fields = [ { 'field': 'DisplayName_Processed', 'type': 'String' }, { 'field': 'Email_Processed', 'type': 'String' }, { 'field': 'Info', 'type': 'String' }, ] deduper = dedupe.Dedupe(fields) deduper.sample(data_d, 15000) if os.path.exists(training_file): print('reading labeled examples from ', training_file) with open(training_file, 'rb') as f: deduper.readTraining(f) print('starting active labeling...') dedupe.consoleLabel(deduper) deduper.train() with open(training_file, 'w') as tf: deduper.writeTraining(tf) with open(settings_file, 'wb') as sf: deduper.writeSettings(sf) threshold = deduper.threshold(data_d, recall_weight=1) print('clustering...') clustered_dupes = deduper.match(data_d, threshold) print('# duplicate sets', len(clustered_dupes)) cluster_membership = {} cluster_id = 0 for (cluster_id, cluster) in enumerate(clustered_dupes): id_set, scores = cluster cluster_d = [data_d[c] for c in id_set] canonical_rep = dedupe.canonicalize(cluster_d) for record_id, score in zip(id_set, scores): cluster_membership[record_id] = { "cluster id": cluster_id, "canonical representation": canonical_rep, "confidence": score } singleton_id = cluster_id + 1 with open(output_file, 'w', encoding="utf-8") as f_output, open(input_file, encoding="utf-8") as f_input: writer = csv.writer(f_output) reader = csv.reader(f_input) heading_row = next(reader) heading_row.insert(0, 'confidence_score') heading_row.insert(0, 'Cluster ID') canonical_keys = canonical_rep.keys() for key in canonical_keys: heading_row.append('canonical_' + key) writer.writerow(heading_row) for row in reader: row_id = row[0] if row_id in cluster_membership: cluster_id = cluster_membership[row_id]["cluster id"] canonical_rep = cluster_membership[row_id][ "canonical representation"] row.insert(0, cluster_membership[row_id]['confidence']) row.insert(0, cluster_id) for key in canonical_keys: row.append(canonical_rep[key].encode('utf8')) else: row.insert(0, None) row.insert(0, singleton_id) singleton_id += 1 for key in canonical_keys: row.append(None) writer.writerow(row) return clustered_dupes
def import_logs(folder, search, keys, time_key="^Time = "): """ keys = {"ExectionTime": ["ExecTime", "ClockTime"]} return a DataFrame Loc, Time KeyName1 Keyname2 1 0.1 0.2 2 """ def find_start(log): """ Fast forward through file till 'Starting time loop' """ for i, line in enumerate(log): if "Starting time loop" in line: return i def extract(line, keys): """ returns key and values as list "ExecutionTime":[0,1] """ import re for key, col_names in keys.items(): if re.search(key, line): return col_names, list( map(float,filter(lambda x: x, re.findall("[0-9\-]+[.]?[0-9]*[e]?[\-\+]?[0-9]*", line)))) return None, None fold, dirs, files = next(os.walk(folder)) logs = [fold + "/" + log for log in files if search in log] p_bar = ProgressBar(n_tot = len(logs)) # Lets make sure that we find Timesteps in the log keys.update({time_key: ['Time']}) for log_number, log_name in enumerate(logs): with open(log_name, encoding="utf-8") as log: f = log.readlines() start = find_start(f) dataDict = defaultdict(list) df=DataFrame() for line in f[start:-1]: col_names, values = extract(line, keys) if not col_names: continue if col_names[0] == 'Time': # a new time step has begun # flush datadict and concat to df # Very slow but, so far the solution # to keep subiterations attached to correct time # FIXME: still needs handling of different length dictionaries df = concat([df, DataFrame(dataDict)]) dataDict = defaultdict(list) for i, col in enumerate(col_names): dataDict[col].append(values[i]) p_bar.next() try: df.index=range(len(df)) df.index.names=['Id'] df['Loc'] = log_number df.set_index('Time', append=True, inplace=True) df.set_index('Loc', append=True, inplace=True) df = df.reorder_levels(['Loc','Time','Id']) p_bar.done() except Exception as e: print(log_name) print("failed to process") print(e) return {}, None return {}, df #DataFrame()
def get_plex_downloads(self): logger.debug("Tautulli PlexTV :: Retrieving current server version.") pms_connect = pmsconnect.PmsConnect() pms_connect.set_server_version() update_channel = pms_connect.get_server_update_channel() logger.debug("Tautulli PlexTV :: Plex update channel is %s." % update_channel) plex_downloads = self.get_plextv_downloads( plexpass=(update_channel == 'beta')) try: available_downloads = json.loads(plex_downloads) except Exception as e: logger.warn( "Tautulli PlexTV :: Unable to load JSON for get_plex_updates.") return {} # Get the updates for the platform pms_platform = common.PMS_PLATFORM_NAME_OVERRIDES.get( plexpy.CONFIG.PMS_PLATFORM, plexpy.CONFIG.PMS_PLATFORM) platform_downloads = available_downloads.get('computer').get(pms_platform) or \ available_downloads.get('nas').get(pms_platform) if not platform_downloads: logger.error( "Tautulli PlexTV :: Unable to retrieve Plex updates: Could not match server platform: %s." % pms_platform) return {} v_old = helpers.cast_to_int("".join( v.zfill(4) for v in plexpy.CONFIG.PMS_VERSION.split('-')[0].split('.')[:4])) v_new = helpers.cast_to_int("".join( v.zfill(4) for v in platform_downloads.get('version', '').split( '-')[0].split('.')[:4])) if not v_old: logger.error( "Tautulli PlexTV :: Unable to retrieve Plex updates: Invalid current server version: %s." % plexpy.CONFIG.PMS_VERSION) return {} if not v_new: logger.error( "Tautulli PlexTV :: Unable to retrieve Plex updates: Invalid new server version: %s." % platform_downloads.get('version')) return {} # Get proper download releases = platform_downloads.get('releases', [{}]) release = next( (r for r in releases if r['distro'] == plexpy.CONFIG.PMS_UPDATE_DISTRO and r['build'] == plexpy.CONFIG.PMS_UPDATE_DISTRO_BUILD), releases[0]) download_info = { 'update_available': v_new > v_old, 'platform': platform_downloads.get('name'), 'release_date': platform_downloads.get('release_date'), 'version': platform_downloads.get('version'), 'requirements': platform_downloads.get('requirements'), 'extra_info': platform_downloads.get('extra_info'), 'changelog_added': platform_downloads.get('items_added'), 'changelog_fixed': platform_downloads.get('items_fixed'), 'label': release.get('label'), 'distro': release.get('distro'), 'distro_build': release.get('build'), 'download_url': release.get('url'), } return download_info
def test_PollMixin_False_then_True(self): i = iter([False, True]) d = self.pm.poll(check_f=lambda: next(i), pollinterval=0.1) return d