def get_user_stats(self, section_id=None): if not session.allow_session_library(section_id): return [] monitor_db = database.MonitorDatabase() user_stats = [] try: if str(section_id).isdigit(): query = 'SELECT (CASE WHEN users.friendly_name IS NULL THEN users.username ' \ 'ELSE users.friendly_name END) AS friendly_name, users.user_id, users.thumb, COUNT(user) AS user_count ' \ 'FROM session_history ' \ 'JOIN session_history_metadata ON session_history_metadata.id = session_history.id ' \ 'JOIN users ON users.user_id = session_history.user_id ' \ 'WHERE section_id = ? ' \ 'GROUP BY users.user_id ' \ 'ORDER BY user_count DESC' result = monitor_db.select(query, args=[section_id]) else: result = [] except Exception as e: logger.warn(u"PlexPy Libraries :: Unable to execute database query for get_user_stats: %s." % e) result = [] for item in result: row = {'friendly_name': item['friendly_name'], 'user_id': item['user_id'], 'user_thumb': item['thumb'], 'total_plays': item['user_count'] } user_stats.append(row) return session.mask_session_info(user_stats, mask_metadata=False)
def discover(self): try: ips = self.filterValidIps(self.__getDnsServerIPs()) logger.info("Found %s dns servers used by host" % (len(ips))) self.serversIpList.extend(ips) except Exception, ex: logger.warn('Failed to get DNS Servers information. %s' % ex)
def verifyDB(localDbClient, dbName): try: returnVal = -1 dbStateQuery = 'SELECT db_name()' debugPrint(4, '[' + SCRIPT_NAME + ':verifyDB] Running query <%s>' % dbStateQuery) dbStateResultSet = doQuery(localDbClient, dbStateQuery) ## Return if query returns no results if dbStateResultSet == None: logger.warn('[' + SCRIPT_NAME + ':verifyDB] Unable to get database name!') return returnVal ## We have query results! while dbStateResultSet.next(): databaseName = dbStateResultSet.getString(1).strip() if databaseName.lower().strip() == dbName.lower().strip(): debugPrint(5, '[' + SCRIPT_NAME + ':verifyDB] Database name <%s> OK' % dbName) returnVal = 1 else: logger.error('[' + SCRIPT_NAME + ':verifyDB] Database name mismatch!! Should be <%s>, got <%s>...' % (dbName, databaseName)) return returnVal return returnVal except: excInfo = logger.prepareJythonStackTrace('') logger.warn('[' + SCRIPT_NAME + ':verifyDB] Exception: <%s>' % excInfo) pass
def discoverReplication(self, mysqlOsh): """ Tries to find config variables related to mysql replication @param ObjectStateHolder mysqlOsh mysql osh @return list list of OSHs """ masterHostIp = self.getProperty('master-host') if not masterHostIp: return if not netutils.isValidIp(masterHostIp): try: resolver = netutils.DnsResolverByShell(self.shell) masterHostIp = resolver.resolveIpsByHostname(masterHostIp)[0] except netutils.ResolveException: logger.warn('Failed to resolve Master Host into IP') return masterPort = self.getProperty('master-port') mysqlReplicationOsh = ObjectStateHolder('mysql_replication') mysqlReplicationOsh.setAttribute('data_name', 'MySQL Replication') mysqlReplicationOsh.setContainer(mysqlOsh) self.setAttribute(mysqlReplicationOsh, 'master_user', self.REPL_ARGS_MAPPING) self.setAttribute(mysqlReplicationOsh, 'master_connect_retry', self.REPL_ARGS_MAPPING) masterHostOsh = modeling.createHostOSH(masterHostIp) serviceAddressOsh = modeling.createServiceAddressOsh(masterHostOsh, masterHostIp, masterPort, modeling.SERVICEADDRESS_TYPE_TCP) clientServerLink = modeling.createLinkOSH('client_server', mysqlReplicationOsh, serviceAddressOsh) clientServerLink.setStringAttribute('clientserver_protocol', 'TCP') clientServerLink.setLongAttribute('clientserver_destport', int(masterPort)) # masterMysqlOsh = modeling.createDatabaseOSH('mysql', 'MySQL. Port ' + masterPort, masterPort, masterHostIp, masterHostOsh) # useLink = modeling.createLinkOSH('use', masterHostOsh, serviceAddressOsh) return [masterHostOsh, serviceAddressOsh, clientServerLink, mysqlReplicationOsh]
def resolveIpFromDns(Framework, ipOrDnsOrAlias, localShell, dnsServers = None): normalizedIp = str(ipOrDnsOrAlias).strip() if not normalizedIp or normalizedIp == "localhost" or (ip_addr.isValidIpAddress(normalizedIp) and (ip_addr.IPAddress(normalizedIp).is_loopback or ip_addr.IPAddress(normalizedIp).is_multicast)): logger.debug('Skipped ip [', normalizedIp, '] for next hop, because it is empty or loopback or not a valid ip address') return None if dnsServers is not None: logger.debug('Trying to resolve ip using provided dnsServers names') dnsResolver = netutils.DNSResolver(localShell) for dnsServer in dnsServers: logger.debug('Trying to resolve ip using DNS Server [', dnsServer, ']') try: resolvedIp = dnsResolver.resolveHostIp(normalizedIp, dnsServer) if resolvedIp is not None: logger.debug('Resolved ip [', resolvedIp, '] from [', normalizedIp, '] using DNS Server [', dnsServer, ']') return resolvedIp except: Framework.reportWarning(logger.prepareJythonStackTrace('')) logger.debug('Failed to resolve [', normalizedIp, ']') try: logger.debug('Trying to resolve ip using local DNS server') resolvedIp = netutils.resolveIP(localShell, normalizedIp) if resolvedIp is not None: logger.debug('Resolved ip [', resolvedIp, '] from [', normalizedIp, '] using configured local DNS Server or hosts file') return resolvedIp else: errorMessage = 'Failed to resolve ip from [' + normalizedIp + '] using configured local DNS Server or hosts file' Framework.reportWarning(errorMessage) logger.warn(errorMessage) except: Framework.reportWarning(logger.prepareJythonStackTrace('')) logger.warn(errorMessage) return resolvedIp
def process_flux(term, work): tag = term[1] logger.detail('Processing ' + tag + ' reaction') label = term[2][1] if label == '': label = default_label(tag + '__') while label in work['reactions'].keys(): newlabel = default_label(tag + '__') logger.warn("Duplicate reaction label '" + label + "', substituting '" + newlabel + "'") label = newlabel # I have no idea whether this is reasonable, but: # outflux reactions have an LHS and can (if necessary) # be given MA rates; influx reactions don't and can't if tag == 'outflux': terms = process_chemterm(term[3], work, -1) rate = process_rateterm(term[4], work, terms) lhs = terms rhs = [] else: terms = process_chemterm(term[3], work, 1) rate = process_rateterm(term[4], work, None) lhs = [] rhs = terms work['reactions'][label] = { 'type' : tag, 'terms' : terms, 'rate' : rate, 'lhs' : lhs, 'rhs': rhs, 'ratespec': term[4] } consolidate_chems(work['reactions'][label])
def process_explicit_rate(term, work, lhs): logger.detail("Processing explicit rateterm") if len(term[2]) > 2: logger.warn("More than 1 rate term supplied, ignoring excess") i_expr, expr, deps = process_mathterm(term[2][1][1], work) return { 'i_expr':i_expr, 'expr':expr, 'depends':deps, 'mathterm':term[2][1][1] }
def finalise_outputs(work): logger.warn(work['outputs'], True) present = [ x for x in work['outputs'] if x in work['symbols']] if present: work['outputs'] = present else: work['outputs'] = work['roots']
def connectToDb(localFramework, ipAddress, dbPort): try: theDbClient = None ## Get protocols protocols = localFramework.getAvailableProtocols(ipAddress, ClientsConsts.SQL_PROTOCOL_NAME) for protocolID in protocols: ## If this protocol entry is not for a Sybase DB, ignore it if localFramework.getProtocolProperty(protocolID, CollectorsConstants.SQL_PROTOCOL_ATTRIBUTE_DBTYPE) != 'Sybase': debugPrint(5, '[' + SCRIPT_NAME + ':DiscoveryMain] Ignoring non Sybase protocol entry...') continue ## Don't bother reconnecting if a connection has already been established if not theDbClient: ## Set DB properties dbConnectionProperties = Properties() dbConnectionProperties.setProperty(CollectorsConstants.PROTOCOL_ATTRIBUTE_PORT, dbPort) # Establish JDBC connection debugPrint(5, '[' + SCRIPT_NAME + ':connectToDb] Attempting connection to CiscoWorks database at port <%s>...' % dbPort) try: theDbClient = localFramework.createClient(protocolID, dbConnectionProperties) except: theDbClient and theDBClient.close() return theDbClient except: excInfo = logger.prepareJythonStackTrace('') logger.warn('[' + SCRIPT_NAME + ':connectToDb] Exception: <%s>' % excInfo) pass
def getConfigsFromHelp(self, fileMonitor): mysqlCommandList = [] for cmd in self.HELP_OPTIONS_LIST: if self.shell.isWinOs(): mysqlCommand = '"%s" %s' % (self.processPath, cmd) else: mysqlCommand = '%s %s' % (self.processPath, cmd) mysqlCommandList.append(mysqlCommand) try: help = self.shell.execAlternateCmdsList(mysqlCommandList) if self.shell.getLastCmdReturnCode() != 0: logger.error('Failed to get MySql help info.') except: logger.error('Failed to get MySql help info. %s' % sys.exc_info()[1]) else: pathsFound = re.search('Default options.*order:.*\n(.*\n)', help) if pathsFound: filesLine = pathsFound.group(1) logger.debug('verifying existence of %s' % filesLine) #suppose that file have to be ended by .ext lookupConfPaths = re.findall('((([a-zA-Z]:)|(~?\/)).*?[\\\/]?\.?\w+\.\w+)(\s|$)', filesLine) if lookupConfPaths: for lookupConfPath in lookupConfPaths: if fileMonitor.checkPath(lookupConfPath[0]): self.configPath = lookupConfPath[0] return self.configPath if not self.configPath: logger.warn('MySQL configuration file was not found in mysql help') return self.configPath
def preparePatterns(excludePatternsList): result = [] if excludePatternsList: patternList = excludePatternsList.split(";") wildcardValidationPattern = Pattern.compile("[\d*?.]+") wildcardSubstitutions = [(Pattern.compile("\."), "\\\\."), (Pattern.compile("\*+"), ".*"), (Pattern.compile("\?"), ".") ] for patternStr in patternList: if patternStr: patternStr = patternStr.strip() wildcardValidationMatcher = wildcardValidationPattern.matcher(String(patternStr)) if wildcardValidationMatcher.matches(): for (rPattern, rStr) in wildcardSubstitutions: rMatcher = rPattern.matcher(String(patternStr)) patternStr = rMatcher.replaceAll(rStr) try: pattern = Pattern.compile(patternStr) result.append(pattern) except: logger.warn("Exception '%s' when compiling pattern '%s', pattern is ignored" % (sys.exc_info()[0], patternStr)) else: logger.warn("Ignoring invalid wildcard pattern '%s'" % patternStr) return result
def getLanguage(framework): language = None try: defaultClient = None try: factory = hyperv.WmiClientFactory(framework) defaultClient = factory.createClient() if defaultClient is not None: wmiProvider = wmiutils.WmiAgentProvider(defaultClient) languageDiscoverer = wmiutils.LanguageDiscoverer(wmiProvider) language = languageDiscoverer.getLanguage() finally: if defaultClient is not None: try: defaultClient.close() except: pass except: logger.warnException("Exception while determining OS language") if language is None: logger.warn("Failed to determine language of target system, default language is used") language = wmiutils.DEFAULT_LANGUAGE return language
def getTopology(self): self.linksMap = defaultdict(list) for (linkMapping, link) in self.__links: targetType = linkMapping.getTargetType() sourceType1 = linkMapping.getSourceEnd1Type() sourceType2 = linkMapping.getSourceEnd2Type() sourceId1 = link.getEnd1Id() sourceId2 = link.getEnd2Id() targetType1 = linkMapping.getTargetEnd1Type() targetType2 = linkMapping.getTargetEnd2Type() isContainer = linkMapping.isContainer() targetEnd1Id = self.__createComplexId(sourceId1, sourceType1, targetType1) targetEnd2Id = self.__createComplexId(sourceId2, sourceType2, targetType2) msg = "%s -- %s --> %s" % (targetEnd1Id, targetType, targetEnd2Id) logger.warn(msg) if self.__hasOsh(targetEnd1Id) and self.__hasOsh(targetEnd2Id): logger.info(msg) (osh1, osh2) = (self.__getOsh(targetEnd1Id), self.__getOsh(targetEnd2Id)) if linkMapping.isReverse(): (osh1, osh2) = (osh2, osh1) link_osh = modeling.createLinkOSH(targetType, osh1, osh2) self.__vector.add(link_osh) self.linksMap[osh1].append(link_osh) self.linksMap[osh2].append(link_osh) if targetType == 'composition' or isContainer: osh2.setContainer(osh1) self.addValidCis() return self.__vector
def open_index(filename, flag='r', useHash=False, mode=0666): if bsddb is None: d = open_anydbm(filename, flag) if not useHash: logger.warn('Falling back to hash index: unable to import bsddb') return d return open_bsddb(filename, flag, useHash, mode)
def get_devices_list(self): devices = self.get_plextv_devices_list(output_format='xml') try: xml_head = devices.getElementsByTagName('Device') except Exception as e: logger.warn(u"PlexPy PlexTV :: Unable to parse XML for get_devices_list: %s." % e) return [] devices_list = [] for a in xml_head: device = {"device_name": helpers.get_xml_attr(a, 'name'), "product": helpers.get_xml_attr(a, 'product'), "product_version": helpers.get_xml_attr(a, 'productVersion'), "platform": helpers.get_xml_attr(a, 'platform'), "platform_version": helpers.get_xml_attr(a, 'platformVersion'), "device": helpers.get_xml_attr(a, 'device'), "model": helpers.get_xml_attr(a, 'model'), "vendor": helpers.get_xml_attr(a, 'vendor'), "provides": helpers.get_xml_attr(a, 'provides'), "device_identifier": helpers.get_xml_attr(a, 'clientIdentifier'), "device_id": helpers.get_xml_attr(a, 'id'), "token": helpers.get_xml_attr(a, 'token') } devices_list.append(device) return devices_list
def action(self, query, args=None, return_last_id=False): if query is None: return with db_lock: sql_result = None attempts = 0 while attempts < 5: try: with self.connection as c: if args is None: sql_result = c.execute(query) else: sql_result = c.execute(query, args) # Our transaction was successful, leave the loop break except sqlite3.OperationalError as e: if "unable to open database file" in e or "database is locked" in e: logger.warn(u"PlexPy Database :: Database Error: %s", e) attempts += 1 time.sleep(1) else: logger.error(u"PlexPy Database :: Database error: %s", e) raise except sqlite3.DatabaseError as e: logger.error(u"PlexPy Database :: Fatal Error executing %s :: %s", query, e) raise return sql_result
def cvapi_check(web=None): import logger if web is None: logger.fdebug('[ComicVine API] ComicVine API Check Running...') if mylar.CVAPI_TIME is None or mylar.CVAPI_TIME == '': c_date = now() c_obj_date = datetime.datetime.strptime(c_date,"%Y-%m-%d %H:%M:%S") mylar.CVAPI_TIME = c_obj_date else: if isinstance(mylar.CVAPI_TIME, unicode): c_obj_date = datetime.datetime.strptime(mylar.CVAPI_TIME,"%Y-%m-%d %H:%M:%S") else: c_obj_date = mylar.CVAPI_TIME if web is None: logger.fdebug('[ComicVine API] API Start Monitoring Time (~15mins): ' + str(mylar.CVAPI_TIME)) now_date = now() n_date = datetime.datetime.strptime(now_date,"%Y-%m-%d %H:%M:%S") if web is None: logger.fdebug('[ComicVine API] Time now: ' + str(n_date)) absdiff = abs(n_date - c_obj_date) mins = round(((absdiff.days * 24 * 60 * 60 + absdiff.seconds) / 60.0),2) if mins < 15: if web is None: logger.info('[ComicVine API] Comicvine API count now at : ' + str(mylar.CVAPI_COUNT) + ' / ' + str(mylar.CVAPI_MAX) + ' in ' + str(mins) + ' minutes.') if mylar.CVAPI_COUNT > mylar.CVAPI_MAX: cvleft = 15 - mins if web is None: logger.warn('[ComicVine API] You have already hit your API limit (' + str(mylar.CVAPI_MAX) + ' with ' + str(cvleft) + ' minutes. Best be slowing down, cowboy.') elif mins > 15: mylar.CVAPI_COUNT = 0 c_date = now() mylar.CVAPI_TIME = datetime.datetime.strptime(c_date,"%Y-%m-%d %H:%M:%S") if web is None: logger.info('[ComicVine API] 15 minute API interval resetting [' + str(mylar.CVAPI_TIME) + ']. Resetting API count to : ' + str(mylar.CVAPI_COUNT)) if web is None: return else: line = str(mylar.CVAPI_COUNT) + ' hits / ' + str(mins) + ' minutes' return line
def getProcessorsByUnitaryComputerSystem(client, unitaryComputerSystem): ''' CimClient, UnitaryComputerSystem -> list(Processor) Get Processors by UnitaryComputerSystem ''' processorInstances = client.getInstances("OMC_Processor") processors = [] ignoredProcessorsCount = 0 for processorInstace in processorInstances: systemName = cim_discover.cleanString(_getCimInstanceProperty(processorInstace, 'SystemName')) # uuid of cpu should match uuid of ESX, safe check if unitaryComputerSystem.name == systemName: processor = Processor() processor.setObjectPath(processorInstace.getObjectPath()) processor.systemName = systemName processor.cpuStatus = cim_discover.getIntFromCimInt(_getCimInstanceProperty(processorInstace, 'CPUStatus')) processor.elementName = cim_discover.cleanString(_getCimInstanceProperty(processorInstace, 'ElementName')) modelName = cim_discover.cleanString(_getCimInstanceProperty(processorInstace, 'ModelName')) processor.modelName = modelName and re.sub(r"\s+", " ", modelName) processor.currentClockSpeed = cim_discover.getIntFromCimInt(_getCimInstanceProperty(processorInstace, 'CurrentClockSpeed')) processor.numberOfEnabledCores = cim_discover.getIntFromCimInt(_getCimInstanceProperty(processorInstace, 'NumberOfEnabledCores')) processor.enabledState = cim_discover.getIntFromCimInt(_getCimInstanceProperty(processorInstace, 'EnabledState')) processors.append(processor) else: ignoredProcessorsCount += 1 if ignoredProcessorsCount > 0: logger.warn("Ignored %s processors due to mismatching UUID or being disabled" % ignoredProcessorsCount) return processors
def getMemoryByUnitaryComputerSystem(client, unitaryComputerSystem): ''' CimClient, UnitaryComputerSystem -> list(Memory) Get Memory by UnitaryComputerSystem ''' memoryInstances = client.getInstances("OMC_Memory") memoryList = [] ignoredMemoryCount = 0 for memoryInstance in memoryInstances: systemName = cim_discover.cleanString(_getCimInstanceProperty(memoryInstance, 'SystemName')) # uuid of memory should match uuid of ESX, safe check if unitaryComputerSystem.name == systemName: memory = Memory() memory.setObjectPath(memoryInstance.getObjectPath()) memory.systemName = systemName memory.numberOfBlocks = cim_discover.getIntFromCimInt(_getCimInstanceProperty(memoryInstance, "NumberOfBlocks")) memory.blockSize = cim_discover.getIntFromCimInt(_getCimInstanceProperty(memoryInstance, "BlockSize")) memoryList.append(memory) else: ignoredMemoryCount += 1 if ignoredMemoryCount > 0: logger.warn("Ignored %s memory instances due to mismatching UUID") return memoryList
def get_filters(self, user_id=None): import urlparse if not user_id: return {} try: monitor_db = database.MonitorDatabase() query = 'SELECT filter_all, filter_movies, filter_tv, filter_music, filter_photos FROM users ' \ 'WHERE user_id = ?' result = monitor_db.select_single(query, args=[user_id]) except Exception as e: logger.warn(u"PlexPy Users :: Unable to execute database query for get_filters: %s." % e) result = {} filters_list = {} for k, v in result.iteritems(): filters = {} for f in v.split('|'): if 'contentRating=' in f or 'label=' in f: filters.update(dict(urlparse.parse_qsl(f))) filters['content_rating'] = tuple(f for f in filters.pop('contentRating', '').split(',') if f) filters['labels'] = tuple(f for f in filters.pop('label', '').split(',') if f) filters_list[k] = filters return filters_list
def onConnection(self, context): ''' Method handles successful connection described by context ''' self.connected = True try: vector = self.discoveryFunction(context, self.framework) if vector is not None: logger.debug(" -- Sending vector of %s objects" % vector.size()) if self._logVector: logger.debug(vector.toXmlString()) self.framework.sendObjects(vector) self.framework.flushObjects() else: logger.warn("Discovery function returned result vector that is None") except JException, ex: msg = ex.getMessage() msg = cim_discover.translateErrorMessage(msg) logger.debug(msg) errormessages.resolveAndReport(msg, cim.Protocol.DISPLAY, self.framework)
def resolveIpFromDnsPortipPortconcepts(Framework, OSHVResult, ipPortconcepts, localShell, dnsServers = None): logger.debug('Resolving concepts') for ipPortConceptEntry in ipPortconcepts.entrySet(): conceptName = ipPortConceptEntry.getKey() conceptFields = ipPortConceptEntry.getValue() logger.debug('processing [', conceptName, ']') PROVIDER_IPs = None PROVIDER_PORTs = None for conceptFieldEntry in conceptFields.entrySet(): fieldName = conceptFieldEntry.getKey().upper() if fieldName == PROVIDER_IP: PROVIDER_IPs = conceptFieldEntry.getValue() elif fieldName == PROVIDER_PORT: PROVIDER_PORTs = conceptFieldEntry.getValue() if PROVIDER_IPs is not None: logger.debug('for concept [', conceptName, '].[', PROVIDER_IP, '] found [', str(len(PROVIDER_IPs)), '] values') if PROVIDER_PORTs is None: processIps(Framework, OSHVResult, PROVIDER_IPs, localShell) elif len(PROVIDER_IPs) != len(PROVIDER_PORTs): errorMessage = 'There is a mismatch between the number of IP addresses and the number of ports that were found. The concept [' + conceptName + '].[' + PROVIDER_IP + '] found [' + str(len(PROVIDER_IPs)) + '] values while for [' + conceptName + '].[' + PROVIDER_PORT + '] found [' + str(len(PROVIDER_PORTs)) + '] values' Framework.reportWarning(errorMessage) logger.warn(errorMessage) processIps(Framework, OSHVResult, PROVIDER_IPs, localShell) else: for index in range(len(PROVIDER_IPs)): resolvedIp = resolveIpFromDns(Framework, PROVIDER_IPs[index], localShell, dnsServers) if resolvedIp is not None: processIpPort(Framework, OSHVResult, resolvedIp, PROVIDER_PORTs[index], localShell, dnsServers) else: logger.error('No ' + PROVIDER_IP + ' field returned for concept [', conceptName, ']')
def getSystemInfo(self): ''' Get general system info (manufacturer, name, model and domain) If host is not in domain, WORKGROUP name will be returned instead of domain. @note: Host name can be shorter than original due to AD restrictions, only 15 symbols. Recommended to use discoverHostInfo method @types: -> HostDo @raise Exception: WMI query failed ''' queryBuilder = self._wmiProvider.getBuilder('Win32_ComputerSystem') queryBuilder.addWmiObjectProperties('Manufacturer', 'Name', 'Model', 'Domain', 'NumberOfProcessors') items = self._wmiProvider.getAgent().getWmiData(queryBuilder) if not items: raise Exception("WMI query failed. No data returned") host = HostDo() system = items[0] # make sure that host name will be in lower case to prevent # from false history changes host.hostName = system.Name.lower() host.hostManufacturer = system.Manufacturer host.hostModel = system.Model if system.Domain: host.osDomain = system.Domain if system.NumberOfProcessors: try: cpuNumber = int(system.NumberOfProcessors.strip()) host.winProcessorsNumber = cpuNumber except: logger.warn('Number of processors value is not an integer' ' type: %s' % system.NumberOfProcessors) return host
def delete_all_history(self, user_id=None): monitor_db = database.MonitorDatabase() try: if str(user_id).isdigit(): logger.info(u"PlexPy Users :: Deleting all history for user id %s from database." % user_id) session_history_media_info_del = \ monitor_db.action('DELETE FROM ' 'session_history_media_info ' 'WHERE session_history_media_info.id IN (SELECT session_history_media_info.id ' 'FROM session_history_media_info ' 'JOIN session_history ON session_history_media_info.id = session_history.id ' 'WHERE session_history.user_id = ?)', [user_id]) session_history_metadata_del = \ monitor_db.action('DELETE FROM ' 'session_history_metadata ' 'WHERE session_history_metadata.id IN (SELECT session_history_metadata.id ' 'FROM session_history_metadata ' 'JOIN session_history ON session_history_metadata.id = session_history.id ' 'WHERE session_history.user_id = ?)', [user_id]) session_history_del = \ monitor_db.action('DELETE FROM ' 'session_history ' 'WHERE session_history.user_id = ?', [user_id]) return 'Deleted all items for user_id %s.' % user_id else: return 'Unable to delete items. Input user_id not valid.' except Exception as e: logger.warn(u"PlexPy Users :: Unable to execute database query for delete_all_history: %s." % e)
def initConnectionConfigurations(self): contextsMap = {} for ip in self.ips: credentialsIdList = self.framework.getAvailableProtocols(ip, VcloudProtocol.SHORT) if not credentialsIdList: logger.warn("No credentials for IP %s found" % ip) msg = errormessages.makeErrorMessage(VcloudProtocol.DISPLAY, None, errormessages.ERROR_NO_CREDENTIALS) connectionContext = ConnectionContext() connectionContext.ipAddress = ip connectionContext.warnings.append(msg) self.connectionHandler.onFailure(connectionContext) continue contextsByCredentialsId = {} for credentialsId in credentialsIdList: connectionContext = ConnectionContext() connectionContext.ipAddress = ip connectionContext.credentialsId = credentialsId contexts = [] self.urlGenerator.generate(connectionContext) for url in self.urlGenerator: connectionContextWithUrl = copy.copy(connectionContext) connectionContextWithUrl.urlString = url contexts.append(connectionContextWithUrl) if contexts: contextsByCredentialsId[credentialsId] = contexts if contextsByCredentialsId: contextsMap[ip] = contextsByCredentialsId self.contextsMap = contextsMap
def getModelAndBiosUuid(self): '''@types: -> HostDo @raise Exception: WMI query failed ''' convertToMicrosoftStandart = GeneralSettingsConfigFile.getInstance().getPropertyStringValue('setBiosUuidToMicrosoftStandart', 'false') hostDo = HostDo() queryBuilder = self._wmiProvider.getBuilder('win32_ComputerSystemProduct') queryBuilder.addWmiObjectProperties('uuid', 'name') computerProductList = self._wmiProvider.getAgent().getWmiData(queryBuilder) for computerProduct in computerProductList: if computerProduct.uuid: if (re.match(r"(0{8}-0{4}-0{4}-0{4}-0{12})", computerProduct.uuid) or re.match(r"([fF]{8}-[fF]{4}-[fF]{4}-[fF]{4}-[fF]{12})", computerProduct.uuid)): logger.debug('Invalid UUID was received. Skipping.') continue if convertToMicrosoftStandart.lower() == 'false': #returned 00010203-0405-0607-0809-0a0b0c0d0e0f #should be 03020100-0504-0706-0809-0a0b0c0d0e0f byteStyle = re.match(r"(\w{2})(\w{2})(\w{2})(\w{2})\-(\w{2})(\w{2})-(\w{2})(\w{2})(.*)", computerProduct.uuid) if byteStyle: group1 = byteStyle.group(4) + byteStyle.group(3) + byteStyle.group(2) + byteStyle.group(1) group2 = byteStyle.group(6) + byteStyle.group(5) group3 = byteStyle.group(8) + byteStyle.group(7) uuidFormated = group1 + '-' + group2 + '-' + group3 + byteStyle.group(9) hostDo.biosUUID = uuidFormated else: logger.warn('UUID is not in proper format.') else: hostDo.biosUUID = computerProduct.uuid logger.warn('BIOS UUID is reported according to Microsoft definitions since parameter setBiosUuidToMicrosoftStandart is set to True.') hostDo.hostModel = computerProduct.name return hostDo
def getVappsForVdc(self, vdcInstance, vcloudClient): from com.vmware.vcloud.sdk import Vapp from com.vmware.vcloud.sdk import VCloudException vappByName = {} _vappRefsByName = vdcInstance.getVappRefsByName() if _vappRefsByName.isEmpty(): return vappByName for vappName in _vappRefsByName.keySet(): reference = _vappRefsByName.get(vappName) if reference: try: vappInstance = Vapp.getVappByReference(vcloudClient, reference) vappResource = vappInstance.getResource() vapp = vcloud.Vapp(vappName) vapp.setInstance(vappInstance) vapp.description = vappResource.getDescription() vapp.isDeployed = vappResource.isDeployed() vapp.status.set(vappResource.getStatus()) vappByName[vappName] = vapp except VCloudException, vex: logger.warn("Failed to retrieve vApp by reference, name = '%s', message = '%s'" % (reference.getName(), vex.getMessage()))
def getCatalogEntriesByTypeForCatalog(self, catalogInstance, vcloudClient): from com.vmware.vcloud.sdk import CatalogItem from com.vmware.vcloud.sdk import VCloudException catalogEntriesByType = {} catalogEntriesByType[VcloudReferenceType.MEDIA] = {} catalogEntriesByType[VcloudReferenceType.VAPP_TEMPLATE] = {} _methodByEntityType = { VcloudReferenceType.MEDIA : self.getMediaByReference, VcloudReferenceType.VAPP_TEMPLATE : self.getVappTemplateByReference } _catalogItemRefs = catalogInstance.getCatalogItemReferences() if _catalogItemRefs.isEmpty(): return catalogEntriesByType for catalogItemReference in _catalogItemRefs: catalogItemInstance = CatalogItem.getCatalogItemByReference(vcloudClient, catalogItemReference) entityReference = catalogItemInstance.getEntityReference() entityType = entityReference.getType() method = _methodByEntityType.get(entityType) if method is not None and entityReference is not None: try: entity = method(entityReference, vcloudClient) entityName = entity.getName() catalogEntriesByType[entityType][entityName] = entity except VCloudException, vex: logger.warn("Failed to retrieve catalog entry by reference, name = '%s', message = '%s'" % (entityReference.getName(), vex.getMessage()))
def getProcesses(self): ''' Get information about all non-system processes -> list(Process) @command: wmic path Win32_Process get CommandLine, CreationDate, ExecutablePath, Name, ProcessId @raise Exception: if WMI query failed ''' queryBuilder = self._provider.getBuilder('Win32_Process') queryBuilder.addWmiObjectProperties('Name', 'ProcessId', 'CommandLine', 'ExecutablePath', 'CreationDate') processes = [] for info in self._provider.getAgent().getWmiData(queryBuilder): name = info.Name pid = info.ProcessId if pid == '-1' or not str(pid).isdigit(): logger.debug("Skip process '%s'. It is system process or has non numeric PID" % name) continue startupTime = info.CreationDate try: startupTime = modeling.getDateFromUtcString(startupTime) except ValueError, ve: logger.warn(str(ve)) startupTime = None cmdline = self.__getCommandLineWithProcessName(info.CommandLine, name) # process argument list argsMatch = re.match('("[^"]+"|[^"]\S+)\s+(.+)$', cmdline) parameters = argsMatch and argsMatch.group(2) or None process = Process(name, cmdline) process.setPid(pid) process.parameters = parameters process.path = info.ExecutablePath processes.append(process)
def __getDnsServerIPs(self): ''' @types: -> list[str] @raise Exception: WMI query failed ''' ips = [] clazz = 'Win32_NetworkAdapterConfiguration' queryBuilder = self._wmiProvider.getBuilder(clazz) queryBuilder.addWmiObjectProperties('dnsServerSearchOrder') queryBuilder.addWhereClause('domainDnsRegistrationEnabled <> NULL') agent = self._wmiProvider.getAgent() dnsServersConfigurationList = agent.getWmiData(queryBuilder) for dnsServersConfiguration in dnsServersConfigurationList: dnsIps = dnsServersConfiguration.dnsServerSearchOrder # depending on protocol this field represented as CSV string # or list of values if not isinstance(dnsIps, types.ListType): dnsIps = map(string.strip, str(dnsIps).split(',')) for ip in dnsIps: if ip: try: if ip_addr.isValidIpAddressNotZero(ip): ips.append(ip_addr.IPAddress(ip)) except: logger.warn('Failed to parse to IP value "%s"' % ip) return ips
def parseResourceGroups(cluster, scrgadmOutput): resourceGroupsByName = {} results = re.findall(r"Res Group name:\s+([\w.-]+)", scrgadmOutput) for groupName in results: group = ResourceGroup(groupName) logger.debug("Found resource group '%s'" % groupName) attributesMap = {} groupAttributeLines = re.findall( r"\(%s\) Res Group([^\n]+)" % re.escape(groupName), scrgadmOutput) for attributeLine in groupAttributeLines: attributeLine = attributeLine.strip() elements = re.split(r":", attributeLine, 1) if len(elements) == 2: attributeName = elements[0] and elements[0].strip() attributeValue = elements[1] and elements[1].strip() if attributeName and attributeValue and attributeValue.lower( ) != '<null>': attributesMap[attributeName] = attributeValue else: logger.warn( "Ignoring invalid resource group attribute line: '%s'" % attributeLine) mode = attributesMap.get('mode') if mode in (ResourceGroup.MODE_FAILOVER, ResourceGroup.MODE_SCALABLE): group.mode = mode maxPrimaries = attributesMap.get('Maximum_primaries') if maxPrimaries: try: group.maxPrimaries = int(maxPrimaries) except: logger.warn( "Failed to convert maximum_primaries value '%s' to integer" % maxPrimaries) desiredPrimaries = attributesMap.get('Desired_primaries') if desiredPrimaries: try: group.desiredPrimaries = int(desiredPrimaries) except: logger.warn( "Failed to convert desired_primaries value '%s' to integer" % desiredPrimaries) managementState = attributesMap.get('management state') if managementState: group.isManaged = managementState.lower() == 'managed' failbackMode = attributesMap.get('Failback') if failbackMode: group.isFailback = failbackMode.lower() == 'true' systemGroup = attributesMap.get('system') if systemGroup: group.isSystem = systemGroup.lower() == 'true' autoStart = attributesMap.get('Auto_start_on_new_cluster') if autoStart: group.autoStartOnNewCluster = autoStart.lower() == 'true' nodeList = attributesMap.get('Nodelist') if nodeList: nodes = re.split(r"\s+", nodeList) for node in nodes: group.configuredNodes.add(node) else: logger.debug( "Resource group '%s' does not have nodes list configured" % groupName) # we assume all nodes in the cluster can run this group for nodeName in cluster.nodesByName.keys(): group.configuredNodes.add(nodeName) resourceGroupsByName[groupName] = group cluster.resourceGroupsByName = resourceGroupsByName
def GetComicInfo(comicid,dom): #comicvine isn't as up-to-date with issue counts.. #so this can get really buggered, really fast. tracks = dom.getElementsByTagName('issue') try: cntit = dom.getElementsByTagName('count_of_issues')[0].firstChild.wholeText except: cntit = len(tracks) trackcnt = len(tracks) logger.fdebug("number of issues I counted: " + str(trackcnt)) logger.fdebug("number of issues CV says it has: " + str(cntit)) # if the two don't match, use trackcnt as count_of_issues might be not upto-date for some reason if int(trackcnt) != int(cntit): cntit = trackcnt vari = "yes" else: vari = "no" logger.fdebug("vari is set to: " + str(vari)) #if str(trackcnt) != str(int(cntit)+2): # cntit = int(cntit) + 1 comic = {} comicchoice = [] cntit = int(cntit) #retrieve the first xml tag (<tag>data</tag>) #that the parser finds with name tagName: # to return the parent name of the <name> node : dom.getElementsByTagName('name')[0].parentNode.nodeName # where [0] denotes the number of the name field(s) # where nodeName denotes the parentNode : ComicName = results, publisher = publisher, issues = issue try: names = len( dom.getElementsByTagName('name') ) n = 0 while ( n < names ): if dom.getElementsByTagName('name')[n].parentNode.nodeName == 'results': try: comic['ComicName'] = dom.getElementsByTagName('name')[n].firstChild.wholeText comic['ComicName'] = comic['ComicName'].rstrip() except: logger.error('There was a problem retrieving the given data from ComicVine. Ensure that www.comicvine.com is accessible AND that you have provided your OWN ComicVine API key.') return elif dom.getElementsByTagName('name')[n].parentNode.nodeName == 'publisher': try: comic['ComicPublisher'] = dom.getElementsByTagName('name')[n].firstChild.wholeText except: comic['ComicPublisher'] = "Unknown" n+=1 except: logger.warn('Something went wrong retrieving from ComicVine. Ensure your API is up-to-date and that comicvine is accessible') return try: comic['ComicYear'] = dom.getElementsByTagName('start_year')[0].firstChild.wholeText except: comic['ComicYear'] = '0000' comic['ComicURL'] = dom.getElementsByTagName('site_detail_url')[trackcnt].firstChild.wholeText desdeck = 0 #the description field actually holds the Volume# - so let's grab it try: descchunk = dom.getElementsByTagName('description')[0].firstChild.wholeText comic_desc = drophtml(descchunk) desdeck +=1 except: comic_desc = 'None' #sometimes the deck has volume labels try: deckchunk = dom.getElementsByTagName('deck')[0].firstChild.wholeText comic_deck = deckchunk desdeck +=1 except: comic_deck = 'None' try: comic['Aliases'] = dom.getElementsByTagName('aliases')[0].firstChild.wholeText #logger.fdebug('Aliases: ' + str(aliases)) except: comic['Aliases'] = 'None' comic['ComicVersion'] = 'noversion' #logger.info('comic_desc:' + comic_desc) #logger.info('comic_deck:' + comic_deck) #logger.info('desdeck: ' + str(desdeck)) while (desdeck > 0): if desdeck == 1: if comic_desc == 'None': comicDes = comic_deck[:30] else: #extract the first 60 characters comicDes = comic_desc[:60].replace('New 52', '') elif desdeck == 2: #extract the characters from the deck comicDes = comic_deck[:30].replace('New 52', '') else: break i = 0 while (i < 2): if 'volume' in comicDes.lower(): #found volume - let's grab it. v_find = comicDes.lower().find('volume') #arbitrarily grab the next 10 chars (6 for volume + 1 for space + 3 for the actual vol #) #increased to 10 to allow for text numbering (+5 max) #sometimes it's volume 5 and ocassionally it's fifth volume. if i == 0: vfind = comicDes[v_find:v_find+15] #if it's volume 5 format basenums = {'zero':'0','one':'1','two':'2','three':'3','four':'4','five':'5','six':'6','seven':'7','eight':'8','nine':'9','ten':'10','i':'1','ii':'2','iii':'3','iv':'4','v':'5'} logger.fdebug('volume X format - ' + str(i) + ': ' + vfind) else: vfind = comicDes[:v_find] # if it's fifth volume format basenums = {'zero':'0','first':'1','second':'2','third':'3','fourth':'4','fifth':'5','sixth':'6','seventh':'7','eighth':'8','nineth':'9','tenth':'10','i':'1','ii':'2','iii':'3','iv':'4','v':'5'} logger.fdebug('X volume format - ' + str(i) + ': ' + vfind) volconv = '' for nums in basenums: if nums in vfind.lower(): sconv = basenums[nums] vfind = re.sub(nums, sconv, vfind.lower()) break #logger.info('volconv: ' + str(volconv)) #now we attempt to find the character position after the word 'volume' if i == 0: volthis = vfind.lower().find('volume') volthis = volthis + 6 # add on the actual word to the position so that we can grab the subsequent digit vfind = vfind[volthis:volthis+4] #grab the next 4 characters ;) elif i == 1: volthis = vfind.lower().find('volume') vfind = vfind[volthis-4:volthis] #grab the next 4 characters ;) if '(' in vfind: #bracket detected in versioning' vfindit = re.findall('[^()]+', vfind) vfind = vfindit[0] vf = re.findall('[^<>]+', vfind) ledigit = re.sub("[^0-9]", "", vf[0]) if ledigit != '': comic['ComicVersion'] = ledigit logger.fdebug("Volume information found! Adding to series record : volume " + comic['ComicVersion']) break i+=1 else: i+=1 if comic['ComicVersion'] == 'noversion': logger.fdebug('comic[ComicVersion]:' + str(comic['ComicVersion'])) desdeck -=1 else: break if vari == "yes": comic['ComicIssues'] = str(cntit) else: comic['ComicIssues'] = dom.getElementsByTagName('count_of_issues')[0].firstChild.wholeText comic['ComicImage'] = dom.getElementsByTagName('super_url')[0].firstChild.wholeText comic['ComicImageALT'] = dom.getElementsByTagName('small_url')[0].firstChild.wholeText comic['FirstIssueID'] = dom.getElementsByTagName('id')[0].firstChild.wholeText # print ("fistIss:" + str(comic['FirstIssueID'])) # comicchoice.append({ # 'ComicName': comic['ComicName'], # 'ComicYear': comic['ComicYear'], # 'Comicid': comicid, # 'ComicURL': comic['ComicURL'], # 'ComicIssues': comic['ComicIssues'], # 'ComicImage': comic['ComicImage'], # 'ComicVolume': ParseVol, # 'ComicPublisher': comic['ComicPublisher'] # }) # comic['comicchoice'] = comicchoice return comic
def checkGithub(auto_update=False): plexpy.COMMITS_BEHIND = 0 # Get the latest version available from github logger.info('Retrieving latest version information from GitHub') url = 'https://api.github.com/repos/%s/%s/commits/%s' % ( plexpy.CONFIG.GIT_USER, plexpy.CONFIG.GIT_REPO, plexpy.CONFIG.GIT_BRANCH) if plexpy.CONFIG.GIT_TOKEN: url = url + '?access_token=%s' % plexpy.CONFIG.GIT_TOKEN version = request.request_json(url, timeout=20, validator=lambda x: type(x) == dict) if version is None: logger.warn( 'Could not get the latest version from GitHub. Are you running a local development version?' ) return plexpy.CURRENT_VERSION plexpy.LATEST_VERSION = version['sha'] logger.debug("Latest version is %s", plexpy.LATEST_VERSION) # See how many commits behind we are if not plexpy.CURRENT_VERSION: logger.info( 'You are running an unknown version of Tautulli. Run the updater to identify your version' ) return plexpy.LATEST_VERSION if plexpy.LATEST_VERSION == plexpy.CURRENT_VERSION: logger.info('Tautulli is up to date') return plexpy.LATEST_VERSION logger.info( 'Comparing currently installed version with latest GitHub version') url = 'https://api.github.com/repos/%s/%s/compare/%s...%s' % ( plexpy.CONFIG.GIT_USER, plexpy.CONFIG.GIT_REPO, plexpy.LATEST_VERSION, plexpy.CURRENT_VERSION) if plexpy.CONFIG.GIT_TOKEN: url = url + '?access_token=%s' % plexpy.CONFIG.GIT_TOKEN commits = request.request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) if commits is None: logger.warn('Could not get commits behind from GitHub.') return plexpy.LATEST_VERSION try: plexpy.COMMITS_BEHIND = int(commits['behind_by']) logger.debug("In total, %d commits behind", plexpy.COMMITS_BEHIND) except KeyError: logger.info( 'Cannot compare versions. Are you running a local development version?' ) plexpy.COMMITS_BEHIND = 0 if plexpy.COMMITS_BEHIND > 0: logger.info('New version is available. You are %s commits behind' % plexpy.COMMITS_BEHIND) url = 'https://api.github.com/repos/%s/%s/releases' % ( plexpy.CONFIG.GIT_USER, plexpy.CONFIG.GIT_REPO) releases = request.request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) if releases is None: logger.warn('Could not get releases from GitHub.') return plexpy.LATEST_VERSION if plexpy.CONFIG.GIT_BRANCH == 'master': release = next((r for r in releases if not r['prerelease']), releases[0]) elif plexpy.CONFIG.GIT_BRANCH == 'beta': release = next( (r for r in releases if not r['tag_name'].endswith('-nightly')), releases[0]) elif plexpy.CONFIG.GIT_BRANCH == 'nightly': release = next((r for r in releases), releases[0]) else: release = releases[0] plexpy.LATEST_RELEASE = release['tag_name'] plexpy.NOTIFY_QUEUE.put({ 'notify_action': 'on_plexpyupdate', 'plexpy_download_info': release, 'plexpy_update_commit': plexpy.LATEST_VERSION, 'plexpy_update_behind': plexpy.COMMITS_BEHIND }) if auto_update: logger.info('Running automatic update.') plexpy.shutdown(restart=True, update=True) elif plexpy.COMMITS_BEHIND == 0: logger.info('Tautulli is up to date') return plexpy.LATEST_VERSION
def get_datatables_user_login(self, user_id=None, jwt_token=None, kwargs=None): default_return = { 'recordsFiltered': 0, 'recordsTotal': 0, 'draw': 0, 'data': [] } if not session.allow_session_user(user_id): return default_return data_tables = datatables.DataTables() if session.get_session_user_id(): custom_where = [[ 'user_login.user_id', session.get_session_user_id() ]] else: custom_where = [['user_login.user_id', user_id]] if user_id else [] columns = [ 'user_login.id AS row_id', 'user_login.timestamp', 'user_login.user_id', 'user_login.user', 'user_login.user_group', 'user_login.ip_address', 'user_login.host', 'user_login.user_agent', 'user_login.success', 'user_login.expiry', 'user_login.jwt_token', '(CASE WHEN users.friendly_name IS NULL OR TRIM(users.friendly_name) = "" \ THEN users.username ELSE users.friendly_name END) AS friendly_name' ] try: query = data_tables.ssp_query( table_name='user_login', columns=columns, custom_where=custom_where, group_by=[], join_types=['LEFT OUTER JOIN'], join_tables=['users'], join_evals=[['user_login.user_id', 'users.user_id']], kwargs=kwargs) except Exception as e: logger.warn( "Tautulli Users :: Unable to execute database query for get_datatables_user_login: %s." % e) return default_return results = query['result'] rows = [] for item in results: (os, browser) = httpagentparser.simple_detect(item['user_agent']) expiry = None current = False if item['jwt_token'] and item['expiry']: _expiry = helpers.iso_to_datetime(item['expiry']) if _expiry > arrow.now(): expiry = _expiry.strftime('%Y-%m-%d %H:%M:%S') current = (item['jwt_token'] == jwt_token) row = { 'row_id': item['row_id'], 'timestamp': item['timestamp'], 'user_id': item['user_id'], 'user_group': item['user_group'], 'ip_address': item['ip_address'], 'host': item['host'], 'user_agent': item['user_agent'], 'os': os, 'browser': browser, 'success': item['success'], 'expiry': expiry, 'current': current, 'friendly_name': item['friendly_name'] or item['user'] } rows.append(row) dict = { 'recordsFiltered': query['filteredCount'], 'recordsTotal': query['totalCount'], 'data': session.friendly_name_to_username(rows), 'draw': query['draw'] } return dict
def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'): """Accept a communicator instance + upload blocks from its upload queue.""" """when inserting a block, we try to upload it to a few peers to add some deniability & increase functionality""" TIMER_NAME = "upload_blocks_from_communicator" session_manager: sessionmanager.BlockUploadSessionManager session_manager = comm_inst.shared_state.get( sessionmanager.BlockUploadSessionManager) tried_peers: UserID = [] finishedUploads = [] comm_inst.blocksToUpload = onionrcrypto.cryptoutils.random_shuffle( comm_inst.blocksToUpload) def remove_from_hidden(bl): sleep(60) try: comm_inst.shared_state.get_by_string( 'PublicAPI').hideBlocks.remove(bl) except ValueError: pass if len(comm_inst.blocksToUpload) != 0: for bl in comm_inst.blocksToUpload: if not stringvalidators.validate_hash(bl): logger.warn('Requested to upload invalid block', terminal=True) comm_inst.decrementThreadCount(TIMER_NAME) return session = session_manager.add_session(bl) for _ in range(min(len(comm_inst.onlinePeers), 6)): try: peer = onlinepeers.pick_online_peer(comm_inst) except onionrexceptions.OnlinePeerNeeded: continue try: session.peer_exists[peer] continue except KeyError: pass try: if session.peer_fails[peer] > 3: continue except KeyError: pass if peer in tried_peers: continue tried_peers.append(peer) url = f'http://{peer}/upload' try: data = block.Block(bl).getRaw() except onionrexceptions.NoDataAvailable: finishedUploads.append(bl) break proxy_type = proxypicker.pick_proxy(peer) logger.info( f"Uploading block {bl[:8]} to {peer}", terminal=True) resp = basicrequests.do_post_request( url, data=data, proxyType=proxy_type, content_type='application/octet-stream') if resp is not False: if resp == 'success': Thread(target=remove_from_hidden, args=[bl], daemon=True).start() session.success() session.peer_exists[peer] = True elif resp == 'exists': session.success() session.peer_exists[peer] = True else: session.fail() session.fail_peer(peer) comm_inst.getPeerProfileInstance(peer).addScore(-5) logger.warn( f'Failed to upload {bl[:8]}, reason: {resp}', terminal=True) else: session.fail() session_manager.clean_session() for x in finishedUploads: try: comm_inst.blocksToUpload.remove(x) comm_inst.shared_state.get_by_string( 'PublicAPI').hideBlocks.remove(x) except ValueError: pass comm_inst.decrementThreadCount(TIMER_NAME)
def executeSoftwareQueryByPath(client, reg_path, prefix=''): ''' Shell, str, str = '' -> list(list(str), str) @command: <prefix>reg query <reg_path>\Uninstall /S @command: reg_mam query <reg_path>\Uninstall /S ''' ntcmdErrStr = 'Remote command returned 1(0x1)' non64BitOsErrStr = 'The system was unable to find the specified registry key or value' queryStr = ' query ' + reg_path + '\Uninstall /S' #First trying the default reg.exe(might not work on Win2k or NT) if len(prefix) > 0 and (not prefix.endswith('\\')): prefix += '\\' cmdRemoteAgent = prefix + 'reg' + queryStr buffer = client.execCmd(cmdRemoteAgent, 120000) #@@CMD_PERMISION ntcmd protocol execution logger.debug('Outputting ', cmdRemoteAgent, ': ...') reg_mamRc = client.getLastCmdReturnCode() if (reg_mamRc != 0) or (buffer.find(ntcmdErrStr) != -1): if (reg_mamRc == 1) and (buffer.find('ERROR: More data is available.') != -1): errMsg = 'reg command returned \'More data is available\' error, not all software might be reported' logger.warn(errMsg) pass else: logger.debug( 'reg query command ended unsuccessfully with return code:%d, error:%s' % (reg_mamRc, buffer)) logger.debug( 'Failed getting software info using default reg.exe trying the reg_mam.exe' ) cmdRemote = 'reg_mam' localFile = CollectorsParameters.BASE_PROBE_MGR_DIR + CollectorsParameters.getDiscoveryResourceFolder( ) + CollectorsParameters.FILE_SEPARATOR + 'reg_mam.exe' remoteFile = client.copyFileIfNeeded(localFile) if not remoteFile: logger.warn('Failed copying %s' % cmdRemote) return [[], ''] cmdRemoteAgent = remoteFile + queryStr buffer = client.execCmd( cmdRemoteAgent, 120000) #@@CMD_PERMISION ntcmd protocol execution regRc = client.getLastCmdReturnCode() if (regRc != 0) or (buffer.find(ntcmdErrStr) != -1): if (regRc == 1) and ( buffer.find('ERROR: More data is available.') != -1): errMsg = 'reg_mam command returned \'More data is available\' error, not all software might be reported' logger.warn(errMsg) pass else: if buffer.find(non64BitOsErrStr) == -1: logger.debug( 'Failed getting software info, reg.exe ended with %d, error:%s' % (regRc, buffer)) return [[], ''] logger.debug('got software buffer from remote registry - parsing...') keys = buffer.split(reg_path) return [keys, buffer]
def install_geoip_db(): maxmind_url = 'http://geolite.maxmind.com/download/geoip/database/' geolite2_gz = 'GeoLite2-City.mmdb.gz' geolite2_md5 = 'GeoLite2-City.md5' geolite2_db = geolite2_gz[:-3] md5_checksum = '' temp_gz = os.path.join(plexpy.CONFIG.CACHE_DIR, geolite2_gz) geolite2_db = plexpy.CONFIG.GEOIP_DB or os.path.join( plexpy.DATA_DIR, geolite2_db) # Retrieve the GeoLite2 gzip file logger.debug( u"Tautulli Helpers :: Downloading GeoLite2 gzip file from MaxMind...") try: maxmind = urllib.URLopener() maxmind.retrieve(maxmind_url + geolite2_gz, temp_gz) md5_checksum = urllib2.urlopen(maxmind_url + geolite2_md5).read() except Exception as e: logger.error( u"Tautulli Helpers :: Failed to download GeoLite2 gzip file from MaxMind: %s" % e) return False # Extract the GeoLite2 database file logger.debug(u"Tautulli Helpers :: Extracting GeoLite2 database...") try: with gzip.open(temp_gz, 'rb') as gz: with open(geolite2_db, 'wb') as db: db.write(gz.read()) except Exception as e: logger.error( u"Tautulli Helpers :: Failed to extract the GeoLite2 database: %s" % e) return False # Check MD5 hash for GeoLite2 database file logger.debug( u"Tautulli Helpers :: Checking MD5 checksum for GeoLite2 database...") try: hash_md5 = hashlib.md5() with open(geolite2_db, 'rb') as f: for chunk in iter(lambda: f.read(4096), b""): hash_md5.update(chunk) md5_hash = hash_md5.hexdigest() if md5_hash != md5_checksum: logger.error( u"Tautulli Helpers :: MD5 checksum doesn't match for GeoLite2 database. " "Checksum: %s, file hash: %s" % (md5_checksum, md5_hash)) return False except Exception as e: logger.error( u"Tautulli Helpers :: Failed to generate MD5 checksum for GeoLite2 database: %s" % e) return False # Delete temportary GeoLite2 gzip file logger.debug( u"Tautulli Helpers :: Deleting temporary GeoLite2 gzip file...") try: os.remove(temp_gz) except Exception as e: logger.warn( u"Tautulli Helpers :: Failed to remove temporary GeoLite2 gzip file: %s" % e) logger.debug( u"Tautulli Helpers :: GeoLite2 database installed successfully.") plexpy.CONFIG.__setattr__('GEOIP_DB', geolite2_db) plexpy.CONFIG.write() return True
def assemble(user_called=True, retry_with_zero_m4v=False): try: ass_json_file = pil.assemble_arg if pil.assemble_arg.endswith( ".json") else pil.assemble_arg + ".json" ass_mp4_file = os.path.join( pil.dl_path, os.path.basename(ass_json_file).replace("_downloads", "").replace( ".json", ".mp4")) ass_segment_dir = pil.assemble_arg if not pil.assemble_arg.endswith( ".json") else pil.assemble_arg.replace(".json", "") if pil.verbose: logger.plain("{}\n{}\n{}".format(ass_json_file, ass_mp4_file, ass_segment_dir)) broadcast_info = {} if not os.path.isdir(ass_segment_dir) or not os.listdir( ass_segment_dir): logger.error( 'The segment directory does not exist or does not contain any files: %s' % ass_segment_dir) logger.separator() return if not os.path.isfile(ass_json_file): logger.warn( "No matching json file found for the segment directory, trying to continue without it." ) ass_stream_id = os.listdir(ass_segment_dir)[0].split('-')[0] broadcast_info['id'] = ass_stream_id broadcast_info['broadcast_status'] = "active" broadcast_info['segments'] = {} else: with open(ass_json_file) as info_file: try: broadcast_info = json.load(info_file) except Exception as e: logger.warn( "Could not decode json file, trying to continue without it." ) ass_stream_id = os.listdir(ass_segment_dir)[0].split( '-')[0] broadcast_info['id'] = ass_stream_id broadcast_info['broadcast_status'] = "active" broadcast_info['segments'] = {} if broadcast_info.get('broadcast_status', '') == 'post_live': logger.error( 'Video segment files from replay downloads cannot be assembled.' ) return stream_id = str(broadcast_info['id']) segment_meta = broadcast_info.get('segments', {}) if segment_meta: all_segments = [ os.path.join(ass_segment_dir, k) for k in broadcast_info['segments'].keys() ] else: all_segments = list( filter( os.path.isfile, glob.glob( os.path.join(ass_segment_dir, '%s-*.m4v' % stream_id)))) all_segments = sorted(all_segments, key=lambda x: _get_file_index(x)) sources = [] audio_stream_format = 'assembled_source_{0}_{1}_mp4.tmp' video_stream_format = 'assembled_source_{0}_{1}_m4a.tmp' video_stream = '' audio_stream = '' has_skipped_zero_m4v = False if not all_segments: logger.error( "No video segment files have been found in the specified folder." ) logger.separator() return else: logger.info( "Assembling video segment files from specified folder: {}". format(ass_segment_dir)) for segment in all_segments: if not os.path.isfile(segment.replace('.m4v', '.m4a')): logger.warn('Audio segment not found: {0!s}'.format( segment.replace('.m4v', '.m4a'))) continue if segment.endswith('-init.m4v'): logger.info('Replacing %s' % segment) segment = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'repair', 'init.m4v') if segment.endswith('-0.m4v') and not retry_with_zero_m4v: has_skipped_zero_m4v = True continue video_stream = os.path.join( ass_segment_dir, video_stream_format.format(stream_id, len(sources))) audio_stream = os.path.join( ass_segment_dir, audio_stream_format.format(stream_id, len(sources))) file_mode = 'ab' with open(video_stream, file_mode) as outfile, open(segment, 'rb') as readfile: shutil.copyfileobj(readfile, outfile) with open(audio_stream, file_mode) as outfile, open( segment.replace('.m4v', '.m4a'), 'rb') as readfile: shutil.copyfileobj(readfile, outfile) if audio_stream and video_stream: sources.append({'video': video_stream, 'audio': audio_stream}) for n, source in enumerate(sources): ffmpeg_binary = os.getenv('FFMPEG_BINARY', 'ffmpeg') cmd = [ ffmpeg_binary, '-loglevel', 'warning', '-y', '-i', source['audio'], '-i', source['video'], '-c:v', 'copy', '-c:a', 'copy', ass_mp4_file ] #fnull = open(os.devnull, 'w') fnull = None exit_code = subprocess.call(cmd, stdout=fnull, stderr=subprocess.STDOUT) if exit_code != 0: logger.warn( "FFmpeg exit code not '0' but '{:d}'.".format(exit_code)) if has_skipped_zero_m4v and not retry_with_zero_m4v: logger.binfo( "*-0.m4v segment was detected but skipped, retrying to assemble video without " "skipping it.") os.remove(source['audio']) os.remove(source['video']) logger.separator() assemble(user_called, retry_with_zero_m4v=True) return else: logger.info('The video file has been generated: %s' % os.path.basename(ass_mp4_file)) os.remove(source['audio']) os.remove(source['video']) if user_called: logger.separator() except Exception as e: logger.error("An error occurred: {:s}".format(str(e)))
def authenticate(username, password, force_use_login_args=False): ig_api = None try: if force_use_login_args: pil.ig_user = username pil.ig_pass = password pil.config_login_overridden = True logger.binfo( "Overriding configuration file login with -u and -p arguments." ) logger.separator() cookie_file = os.path.join(os.path.dirname(pil.config_path), "{}.json".format(username)) if not os.path.isfile(cookie_file): # settings file does not exist logger.warn('Unable to find cookie file: {0!s}'.format( os.path.basename(cookie_file))) logger.info('Creating a new one.') # login new ig_api = Client( username, password, on_login=lambda x: onlogin_callback(x, cookie_file), proxy=pil.proxy) # ig_api = Client(username, password, on_login=lambda x: onlogin_callback(x, cookie_file), proxy=pil.proxy) login(ig_api) else: with open(cookie_file) as file_data: cached_settings = json.load(file_data, object_hook=from_json) logger.info('Using settings file: {0!s}'.format(cookie_file)) # device_id = cached_settings.get('device_id') # reuse auth cached_settings try: ig_api = Client(username, password, settings=cached_settings, proxy=pil.proxy) except (ClientSentryBlockError, ClientChallengeRequiredError, ClientCheckpointRequiredError, ClientCookieExpiredError, ClientLoginError, ClientError) as e: logger.separator() logger.warn('Some sort of login exception!') if pil.verbose: logger.plain(json.dumps(e.error_response)) logger.error('Could not login: {:s}'.format(e.error_response)) logger.error('{:s}'.format( json.loads(e.error_response).get("message", e.error_response))) logger.error('{:s}'.format(e.error_response)) logger.separator() ig_api = Client( username, password, on_login=lambda x: onlogin_callback(x, cookie_file), proxy=pil.proxy) login(ig_api) logger.warn('successfully resolved error and logged back in!') except (ClientLoginError, ClientError) as e: logger.separator() if pil.verbose: logger.plain(json.dumps(e.error_response)) logger.error('Could not login: {:s}'.format(e.error_response)) logger.error('{:s}'.format( json.loads(e.error_response).get("message", e.error_response))) logger.error('{:s}'.format(e.error_response)) logger.separator() except Exception as e: if pil.verbose: logger.plain(json.dumps(e)) if str(e).startswith("unsupported pickle protocol"): logger.warn( "This cookie file is not compatible with Python {}.".format( sys.version.split(' ')[0][0])) logger.warn( "Please delete your cookie file '{}.json' and try again.". format(username)) else: logger.separator() logger.error('Unexpected exception: {:s}'.format(e)) logger.separator() except KeyboardInterrupt: logger.separator() logger.warn("The user authentication has been aborted.") logger.separator() if ig_api: logger.info('Successfully logged into account: {:s}'.format( str(ig_api.authenticated_user_name))) if pil.show_cookie_expiry and not force_use_login_args: try: cookie_expiry = ig_api.cookie_jar.auth_expires logger.info('Cookie file expiry date: {:s}'.format( datetime.datetime.fromtimestamp(cookie_expiry).strftime( '%Y-%m-%d at %I:%M:%S %p'))) except Exception as e: logger.warn( 'An error occurred while getting the cookie file expiry date: {:s}' .format(str(e))) logger.separator() return ig_api else: return None
def startTor(self): ''' Start Tor with onion service on port 80 & socks proxy on random port ''' self.generateTorrc() if os.path.exists('./tor'): torBinary = './tor' elif os.path.exists('/usr/bin/tor'): torBinary = '/usr/bin/tor' else: torBinary = 'tor' try: tor = subprocess.Popen([torBinary, '-f', self.torConfigLocation], stdout=subprocess.PIPE, stderr=subprocess.PIPE) except FileNotFoundError: logger.fatal( "Tor was not found in your path or the Onionr directory. Please install Tor and try again." ) sys.exit(1) else: # Test Tor Version torVersion = subprocess.Popen([torBinary, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) for line in iter(torVersion.stdout.readline, b''): if 'Tor 0.2.' in line.decode(): logger.warn( "Running 0.2.x Tor series, no support for v3 onion peers" ) break torVersion.kill() # wait for tor to get to 100% bootstrap try: for line in iter(tor.stdout.readline, b''): if 'Bootstrapped 100%: Done' in line.decode(): break elif 'Opening Socks listener' in line.decode(): logger.debug(line.decode().replace('\n', '')) else: logger.fatal( 'Failed to start Tor. Try killing any other Tor processes owned by this user.' ) return False except KeyboardInterrupt: logger.fatal("Got keyboard interrupt") return False logger.info('Finished starting Tor', timestamp=True) self.readyState = True myID = open('data/hs/hostname', 'r') self.myID = myID.read().replace('\n', '') myID.close() torPidFile = open('data/torPid.txt', 'w') torPidFile.write(str(tor.pid)) torPidFile.close() return True
def run(): from websocket import create_connection if plexpy.CONFIG.PMS_SSL and plexpy.CONFIG.PMS_URL[:5] == 'https': uri = plexpy.CONFIG.PMS_URL.replace( 'https://', 'wss://') + '/:/websockets/notifications' secure = ' secure' else: uri = 'ws://%s:%s/:/websockets/notifications' % ( plexpy.CONFIG.PMS_IP, plexpy.CONFIG.PMS_PORT) secure = '' # Set authentication token (if one is available) if plexpy.CONFIG.PMS_TOKEN: header = ["X-Plex-Token: %s" % plexpy.CONFIG.PMS_TOKEN] else: header = [] global ws_reconnect ws_reconnect = False ws_connected = False reconnects = 0 # Try an open the websocket connection - if it fails after 15 retries fallback to polling while not ws_connected and reconnects <= 15: try: logger.info( u"PlexPy WebSocket :: Opening%s websocket, connection attempt %s." % (secure, str(reconnects + 1))) ws = create_connection(uri, header=header) reconnects = 0 ws_connected = True logger.info(u"PlexPy WebSocket :: Ready") except IOError as e: logger.error(u"PlexPy WebSocket :: %s." % e) reconnects += 1 time.sleep(5) while ws_connected: try: process(*receive(ws)) # successfully received data, reset reconnects counter reconnects = 0 except (websocket.WebSocketConnectionClosedException, Exception): if reconnects <= 15: reconnects += 1 # Sleep 5 between connection attempts if reconnects > 1: time.sleep(5) logger.warn( u"PlexPy WebSocket :: Connection has closed, reconnection attempt %s." % reconnects) try: ws = create_connection(uri, header=header) except IOError as e: logger.info(u"PlexPy WebSocket :: %s." % e) else: ws.shutdown() ws_connected = False break # Check if we recieved a restart notification and close websocket connection cleanly if ws_reconnect: logger.info(u"PlexPy WebSocket :: Reconnecting websocket...") ws.shutdown() ws_connected = False start_thread() if not ws_connected and not ws_reconnect: logger.error( u"PlexPy WebSocket :: Connection unavailable, falling back to polling." ) plexpy.POLLING_FAILOVER = True plexpy.initialize_scheduler() logger.debug(u"PlexPy WebSocket :: Leaving thread.")
def processor(self): sendresponse = self.params['nzo_id'] try: logger.info('sending now to %s' % self.sab_url) logger.info('parameters set to %s' % self.params) time.sleep( 5 ) #pause 5 seconds before monitoring just so it hits the queue h = requests.get(self.sab_url, params=self.params['queue'], verify=False) except Exception as e: logger.info('uh-oh: %s' % e) return {'status': False} else: queueresponse = h.json() logger.info('successfully queried the queue for status') try: queueinfo = queueresponse['queue'] logger.info('queue: %s' % queueresponse) logger.info('Queue status : %s' % queueinfo['status']) logger.info('Queue mbleft : %s' % queueinfo['mbleft']) while any([ str(queueinfo['status']) == 'Downloading', str(queueinfo['status']) == 'Idle' ]) and float(queueinfo['mbleft']) > 0: logger.info('queue_params: %s' % self.params['queue']) queue_resp = requests.get(self.sab_url, params=self.params['queue'], verify=False) queueresp = queue_resp.json() queueinfo = queueresp['queue'] logger.info('status: %s' % queueinfo['status']) logger.info('mbleft: %s' % queueinfo['mbleft']) logger.info('timeleft: %s' % queueinfo['timeleft']) logger.info('eta: %s' % queueinfo['eta']) time.sleep(5) except Exception as e: logger.warn('error: %s' % e) logger.info('File has now downloaded!') hist_params = { 'mode': 'history', 'category': mylar.CONFIG.SAB_CATEGORY, 'failed': 0, 'output': 'json', 'apikey': mylar.CONFIG.SAB_APIKEY } hist = requests.get(self.sab_url, params=hist_params, verify=False) historyresponse = hist.json() #logger.info(historyresponse) histqueue = historyresponse['history'] found = {'status': False} while found['status'] is False: try: for hq in histqueue['slots']: #logger.info('nzo_id: %s --- %s [%s]' % (hq['nzo_id'], sendresponse, hq['status'])) if hq['nzo_id'] == sendresponse and hq[ 'status'] == 'Completed': logger.info( 'found matching completed item in history. Job has a status of %s' % hq['status']) if os.path.isfile(hq['storage']): logger.info('location found @ %s' % hq['storage']) found = { 'status': True, 'name': re.sub('.nzb', '', hq['nzb_name']).strip(), 'location': os.path.abspath( os.path.join(hq['storage'], os.pardir)), 'failed': False } break else: logger.info( 'no file found where it should be @ %s - is there another script that moves things after completion ?' % hq['storage']) break elif hq['nzo_id'] == sendresponse and hq[ 'status'] == 'Failed': #get the stage / error message and see what we can do stage = hq['stage_log'] for x in stage[0]: if 'Failed' in x['actions'] and any([ x['name'] == 'Unpack', x['name'] == 'Repair' ]): if 'moving' in x['actions']: logger.warn( 'There was a failure in SABnzbd during the unpack/repair phase that caused a failure: %s' % x['actions']) else: logger.warn( 'Failure occured during the Unpack/Repair phase of SABnzbd. This is probably a bad file: %s' % x['actions']) if mylar.FAILED_DOWNLOAD_HANDLING is True: found = { 'status': True, 'name': re.sub('.nzb', '', hq['nzb_name']).strip(), 'location': os.path.abspath( os.path.join( hq['storage'], os.pardir)), 'failed': True } break break except Exception as e: logger.warn('error %s' % e) break return found
def get_datatables_list(self, kwargs=None, grouping=None): default_return = { 'recordsFiltered': 0, 'recordsTotal': 0, 'draw': 0, 'data': [] } data_tables = datatables.DataTables() custom_where = [['users.deleted_user', 0]] if grouping is None: grouping = plexpy.CONFIG.GROUP_HISTORY_TABLES if session.get_session_user_id(): custom_where.append( ['users.user_id', session.get_session_user_id()]) if kwargs.get('user_id'): custom_where.append(['users.user_id', kwargs.get('user_id')]) group_by = 'session_history.reference_id' if grouping else 'session_history.id' columns = [ 'users.id AS row_id', 'users.user_id', 'users.username', '(CASE WHEN users.friendly_name IS NULL OR TRIM(users.friendly_name) = "" \ THEN users.username ELSE users.friendly_name END) AS friendly_name', 'users.thumb AS user_thumb', 'users.custom_avatar_url AS custom_thumb', 'COUNT(DISTINCT %s) AS plays' % group_by, 'SUM(CASE WHEN session_history.stopped > 0 THEN (session_history.stopped - session_history.started) \ ELSE 0 END) - SUM(CASE WHEN session_history.paused_counter IS NULL THEN 0 ELSE \ session_history.paused_counter END) AS duration', 'MAX(session_history.started) AS last_seen', 'MAX(session_history.id) AS history_row_id', 'session_history_metadata.full_title AS last_played', 'session_history.ip_address', 'session_history.platform', 'session_history.player', 'session_history.rating_key', 'session_history_metadata.media_type', 'session_history_metadata.thumb', 'session_history_metadata.parent_thumb', 'session_history_metadata.grandparent_thumb', 'session_history_metadata.parent_title', 'session_history_metadata.year', 'session_history_metadata.media_index', 'session_history_metadata.parent_media_index', 'session_history_metadata.live', 'session_history_metadata.added_at', 'session_history_metadata.originally_available_at', 'session_history_metadata.guid', 'session_history_media_info.transcode_decision', 'users.do_notify AS do_notify', 'users.keep_history AS keep_history', 'users.allow_guest AS allow_guest', 'users.is_active AS is_active' ] try: query = data_tables.ssp_query( table_name='users', columns=columns, custom_where=custom_where, group_by=['users.user_id'], join_types=[ 'LEFT OUTER JOIN', 'LEFT OUTER JOIN', 'LEFT OUTER JOIN' ], join_tables=[ 'session_history', 'session_history_metadata', 'session_history_media_info' ], join_evals=[ ['session_history.user_id', 'users.user_id'], ['session_history.id', 'session_history_metadata.id'], ['session_history.id', 'session_history_media_info.id'] ], kwargs=kwargs) except Exception as e: logger.warn( "Tautulli Users :: Unable to execute database query for get_list: %s." % e) return default_return users = query['result'] rows = [] for item in users: if item['media_type'] == 'episode' and item['parent_thumb']: thumb = item['parent_thumb'] elif item['media_type'] == 'episode': thumb = item['grandparent_thumb'] else: thumb = item['thumb'] if item['custom_thumb'] and item['custom_thumb'] != item[ 'user_thumb']: user_thumb = item['custom_thumb'] elif item['user_thumb']: user_thumb = item['user_thumb'] else: user_thumb = common.DEFAULT_USER_THUMB # Rename Mystery platform names platform = common.PLATFORM_NAME_OVERRIDES.get( item['platform'], item['platform']) row = { 'row_id': item['row_id'], 'user_id': item['user_id'], 'username': item['username'], 'friendly_name': item['friendly_name'], 'user_thumb': user_thumb, 'plays': item['plays'], 'duration': item['duration'], 'last_seen': item['last_seen'], 'last_played': item['last_played'], 'history_row_id': item['history_row_id'], 'ip_address': item['ip_address'], 'platform': platform, 'player': item['player'], 'rating_key': item['rating_key'], 'media_type': item['media_type'], 'thumb': thumb, 'parent_title': item['parent_title'], 'year': item['year'], 'media_index': item['media_index'], 'parent_media_index': item['parent_media_index'], 'live': item['live'], 'originally_available_at': item['originally_available_at'], 'guid': item['guid'], 'transcode_decision': item['transcode_decision'], 'do_notify': helpers.checked(item['do_notify']), 'keep_history': helpers.checked(item['keep_history']), 'allow_guest': helpers.checked(item['allow_guest']), 'is_active': item['is_active'] } rows.append(row) dict = { 'recordsFiltered': query['filteredCount'], 'recordsTotal': query['totalCount'], 'data': session.friendly_name_to_username(rows), 'draw': query['draw'] } return dict
def _debug_execution_exception(e): logger.warn(str(e)) if e.result and e.result.exception: logger.warn(e.result.exception.getMessage())
def daemon(o_inst): ''' Starts the Onionr communication daemon ''' # remove runcheck if it exists if os.path.isfile('%s/.runcheck' % (o_inst.onionrCore.dataDir,)): logger.debug('Runcheck file found on daemon start, deleting in advance.') os.remove('%s/.runcheck' % (o_inst.onionrCore.dataDir,)) Thread(target=api.API, args=(o_inst, o_inst.debug, onionr.API_VERSION)).start() Thread(target=api.PublicAPI, args=[o_inst.getClientApi()]).start() try: time.sleep(0) except KeyboardInterrupt: logger.debug('Got keyboard interrupt, shutting down...') _proper_shutdown(o_inst) apiHost = '' while apiHost == '': try: with open(o_inst.onionrCore.publicApiHostFile, 'r') as hostFile: apiHost = hostFile.read() except FileNotFoundError: pass time.sleep(0.5) #onionr.Onionr.setupConfig('data/', self = o_inst) if o_inst._developmentMode: logger.warn('DEVELOPMENT MODE ENABLED (NOT RECOMMENDED)', timestamp = False) net = NetController(o_inst.onionrCore.config.get('client.public.port', 59497), apiServerIP=apiHost) logger.debug('Tor is starting...') if not net.startTor(): o_inst.onionrUtils.localCommand('shutdown') sys.exit(1) if len(net.myID) > 0 and o_inst.onionrCore.config.get('general.security_level', 1) == 0: logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID)) else: logger.debug('.onion service disabled') logger.debug('Using public key: %s' % (logger.colors.underline + o_inst.onionrCore._crypto.pubKey)) try: time.sleep(1) except KeyboardInterrupt: _proper_shutdown(o_inst) o_inst.onionrCore.torPort = net.socksPort communicatorThread = Thread(target=communicator.startCommunicator, args=(o_inst, str(net.socksPort))) communicatorThread.start() while o_inst.communicatorInst is None: time.sleep(0.1) # print nice header thing :) if o_inst.onionrCore.config.get('general.display_header', True): o_inst.header() # print out debug info o_inst.version(verbosity = 5, function = logger.debug) logger.debug('Python version %s' % platform.python_version()) logger.debug('Started communicator.') events.event('daemon_start', onionr = o_inst) while True: try: time.sleep(3) except KeyboardInterrupt: o_inst.communicatorInst.shutdown = True finally: # Debug to print out used FDs (regular and net) #proc = psutil.Process() #print('api-files:',proc.open_files(), len(psutil.net_connections())) # Break if communicator process ends, so we don't have left over processes if o_inst.communicatorInst.shutdown: break if o_inst.killed: break # Break out if sigterm for clean exit signal.signal(signal.SIGINT, _ignore_sigint) o_inst.onionrCore.daemonQueueAdd('shutdown') o_inst.onionrUtils.localCommand('shutdown') net.killTor() time.sleep(3) o_inst.deleteRunFiles() return
def main(): parser = argparse.ArgumentParser(description='Script to create flux maps.') parser.add_argument( 'inputfile', help='''Simulation results to use as input. ''' '''Supports retrieving files from EOS via the XRootD protocol.''') parser.add_argument( '-o', '--outputfile', default='flux_map.root', help='''File to write the flux maps to. ''' '''Will be recreated if it already exists.''') args = parser.parse_args() f = r.TFile.Open(args.outputfile, 'recreate') f.cd() maxpt = 10. * u.GeV maxp = 360. * u.GeV h = {} # Define histograms for nplane in range(0, 23): ut.bookHist(h, 'NuTauMu_all_{}'.format(nplane), 'Rpc_{};x[cm];y[cm]'.format( nplane), 100, -300, +300, 100, -300, 300) ut.bookHist(h, 'NuTauMu_mu_{}'.format(nplane), 'Rpc_{};x[cm];y[cm]'.format( nplane), 100, -300, +300, 100, -300, 300) for suffix, title in [('mu', '#mu#pm hits'), ('all', 'All hits')]: ut.bookHist(h, 'muon_tiles_{}'.format(suffix), '{};x[cm];y[cm]'.format(title), 200, -1000, +1000, 90, -900, 900) ut.bookHist(h, 'muon_bars_x_{}'.format(suffix), '{};x[cm];y[cm]'.format(title), 2, -300, +300, 240, -600, 600) ut.bookHist(h, 'muon_bars_y_{}'.format(suffix), '{};x[cm];y[cm]'.format(title), 120, -300, +300, 4, -600, 600) ut.bookHist(h, 'timing_{}'.format(suffix), '{};x[cm];y[cm]'.format(title), 3, -252, +252, 167, -501, 501) ut.bookHist(h, 'TargetTracker_{}'.format(suffix), '{};x[cm];y[cm]'.format(title), 120, -60, 60, 120, -60, 60) ut.bookHist(h, 'TargetTracker_yvsz_{}'.format(suffix), '{};z[cm];y[cm]'.format( title), 400, -3300, -2900, 120, -60, 60) ut.bookHist(h, 'TargetTracker_xvsz_{}'.format(suffix), '{};z[cm];x[cm]'.format( title), 400, -3300, -2900, 120, -60, 60) ut.bookHist(h, 'NuTauMu_{}'.format(suffix), '{};x[cm];y[cm]'.format(title), 100, -300, +300, 100, -300, 300) ut.bookHist(h, 'NuTauMu_yvsz_{}'.format(suffix), '{};z[cm];y[cm]'.format( title), 200, -2680, -2480, 600, -300, 300) ut.bookHist(h, 'NuTauMu_xvsz_{}'.format(suffix), '{};z[cm];x[cm]'.format( title), 200, -2680, -2480, 600, -300, 300) ut.bookHist(h, 'ECAL_TP_{}'.format(suffix), '{};x[cm];y[cm]'.format(title), 167, -501, +501, 334, -1002, 1002) ut.bookHist(h, 'ECAL_Alt_{}'.format(suffix), '{};x[cm];y[cm]'.format(title), 50, -500, +500, 100, -1000, 1000) ut.bookHist(h, 'SBT_Liquid_{}'.format(suffix), '{};z[cm];#phi'.format(title), 100, -3000, +3000, 100, -r.TMath.Pi(), r.TMath.Pi()) ut.bookHist(h, 'SBT_Plastic_{}'.format(suffix), '{};z[cm];#phi'.format(title), 100, -3000, +3000, 100, -r.TMath.Pi(), r.TMath.Pi()) for station in range(1, 5): ut.bookHist(h, 'T{}_{}'.format(station, suffix), '{};x[cm];y[cm]'.format(title), 10, -250, +250, 20, -500, 500) ut.bookHist(h, 'NuTauMu_mu_p', '#mu#pm;p[GeV];', 100, 0, maxp) ut.bookHist(h, 'NuTauMu_mu_pt', '#mu#pm;p_t[GeV];', 100, 0, maxpt) ut.bookHist(h, 'NuTauMu_mu_ppt', '#mu#pm;p[GeV];p_t[GeV];', 100, 0, maxp, 100, 0, maxpt) ut.bookHist(h, 'NuTauMu_all_p', '#mu#pm;p[GeV];', 100, 0, maxp) ut.bookHist(h, 'NuTauMu_all_pt', '#mu#pm;p_t[GeV];', 100, 0, maxpt) ut.bookHist(h, 'NuTauMu_all_ppt', '#mu#pm;p[GeV];p_t[GeV];', 100, 0, maxp, 100, 0, maxpt) for suffix in ['', '_original']: ut.bookHist(h, 'mu_p{}'.format(suffix), '#mu#pm;p[GeV];', 100, 0, maxp) ut.bookHist(h, 'mu_pt{}'.format(suffix), '#mu#pm;p_t[GeV];', 100, 0, maxpt) ut.bookHist(h, 'mu_ppt{}'.format(suffix), '#mu#pm;p[GeV];p_t[GeV];', 100, 0, maxp, 100, 0, maxpt) ut.bookHist(h, 'ECAL_TP_e', 'e#pm with E#geq 250 MeV;x[cm];y[cm]', 167, -501, +501, 334, -1002, 1002) ut.bookHist(h, 'ECAL_Alt_e', 'e#pm with E#geq 250 MeV;x[cm];y[cm]', 50, -500, +500, 100, -1000, 1000) ut.bookHist(h, 'ECAL_TP_gamma', '#gamma;x[cm];y[cm]', 167, -501, +501, 334, -1002, 1002) ut.bookHist(h, 'ECAL_Alt_gamma', '#gamma;x[cm];y[cm]', 50, -500, +500, 100, -1000, 1000) ut.bookHist(h, 'ECAL_e_E', 'e#pm;E[GeV/c^{2}];', 100, 0, 1) ut.bookHist(h, 'ECAL_gamma_E', '#gamma;E[GeV/c^{2}];', 100, 0, 1) ch = r.TChain('cbmsim') ch.Add(args.inputfile) n = ch.GetEntries() log.info(n) i = 0 for event in ch: if i % 10000 == 0: log.info('{}/{}'.format(i, n)) i += 1 muon = False for hit in event.strawtubesPoint: if hit: if not hit.GetEnergyLoss() > 0: continue trid = hit.GetTrackID() assert trid > 0 weight = event.MCTrack[trid].GetWeight() x = hit.GetX() y = hit.GetY() z = hit.GetZ() px = hit.GetPx() py = hit.GetPy() pz = hit.GetPz() pt = np.hypot(px, py) P = np.hypot(pz, pt) pid = hit.PdgCode() assert pid not in [12, -12, 14, -14, 16, -16] detector_ID = hit.GetDetectorID() station = detector_ID // 10000000 h['T{}_all'.format(station)].Fill(x, y, weight) if abs(pid) == 13: muon = True muonid = trid h['T{}_mu'.format(station)].Fill(x, y, weight) h['mu_p'].Fill(P, weight) h['mu_pt'].Fill(pt, weight) h['mu_ppt'].Fill(P, pt, weight) for hit in event.EcalPoint: if hit: if not hit.GetEnergyLoss() > 0: continue trid = hit.GetTrackID() assert trid > 0 weight = event.MCTrack[trid].GetWeight() x = hit.GetX() y = hit.GetY() px = hit.GetPx() py = hit.GetPy() pz = hit.GetPz() pt = np.hypot(px, py) P = np.hypot(pz, pt) pid = hit.PdgCode() if pid in [12, -12, 14, -14, 16, -16]: continue h['ECAL_TP_all'].Fill(x, y, weight) h['ECAL_Alt_all'].Fill(x, y, weight) if abs(pid) == 13: muon = True muonid = trid h['mu_p'].Fill(P, weight) h['mu_pt'].Fill(pt, weight) h['mu_ppt'].Fill(P, pt, weight) h['ECAL_TP_mu'].Fill(x, y, weight) h['ECAL_Alt_mu'].Fill(x, y, weight) elif abs(pid) == 11: Esq = px ** 2 + py ** 2 + pz ** 2 + 0.000511 ** 2 h['ECAL_e_E'].Fill(np.sqrt(Esq), weight) h['ECAL_TP_e'].Fill(x, y, weight) h['ECAL_Alt_e'].Fill(x, y, weight) elif abs(pid) == 22: Esq = px ** 2 + py ** 2 + pz ** 2 h['ECAL_gamma_E'].Fill(np.sqrt(Esq), weight) h['ECAL_TP_gamma'].Fill(x, y, weight) h['ECAL_Alt_gamma'].Fill(x, y, weight) for hit in event.TTPoint: if hit: if not hit.GetEnergyLoss() > 0: continue trid = hit.GetTrackID() assert trid > 0 detID = hit.GetDetectorID() weight = event.MCTrack[trid].GetWeight() x = hit.GetX() y = hit.GetY() z = hit.GetZ() px = hit.GetPx() py = hit.GetPy() pz = hit.GetPz() pt = np.hypot(px, py) P = np.hypot(pz, pt) pid = hit.PdgCode() assert pid not in [12, -12, 14, -14, 16, -16] if detID == 0: h['TargetTracker_all'].Fill(x, y, weight) h['TargetTracker_xvsz_all'].Fill(z, x, weight) h['TargetTracker_yvsz_all'].Fill(z, y, weight) if abs(pid) == 13: muon = True muonid = trid h['mu_p'].Fill(P, weight) h['mu_pt'].Fill(pt, weight) h['mu_ppt'].Fill(P, pt, weight) if detID == 0: h['TargetTracker_mu'].Fill(x, y, weight) h['TargetTracker_xvsz_mu'].Fill(z, x, weight) h['TargetTracker_yvsz_mu'].Fill(z, y, weight) for hit in event.ShipRpcPoint: if hit: if not hit.GetEnergyLoss() > 0: continue trid = hit.GetTrackID() assert trid > 0 detID = hit.GetDetectorID() nplane = detID - 10000 weight = event.MCTrack[trid].GetWeight() x = hit.GetX() y = hit.GetY() z = hit.GetZ() px = hit.GetPx() py = hit.GetPy() pz = hit.GetPz() pt = np.hypot(px, py) P = np.hypot(pz, pt) pid = hit.PdgCode() assert pid not in [12, -12, 14, -14, 16, -16] h['NuTauMu_all'].Fill(x, y, weight) if nplane >= 0: h['NuTauMu_all_{}'.format(nplane)].Fill(x, y, weight) h['NuTauMu_xvsz_all'].Fill(z, x, weight) h['NuTauMu_yvsz_all'].Fill(z, y, weight) if detID == 10000: h['NuTauMu_all_p'].Fill(P, weight) h['NuTauMu_all_pt'].Fill(pt, weight) h['NuTauMu_all_ppt'].Fill(P, pt, weight) if abs(pid) == 13: muon = True muonid = trid h['mu_p'].Fill(P, weight) h['mu_pt'].Fill(pt, weight) h['mu_ppt'].Fill(P, pt, weight) h['NuTauMu_mu'].Fill(x, y, weight) if nplane >= 0: # fill the histogram corresponding to nplane h['NuTauMu_mu_{}'.format(nplane)].Fill(x, y, weight) if detID == 10000: h['NuTauMu_mu_p'].Fill(P, weight) h['NuTauMu_mu_pt'].Fill(pt, weight) h['NuTauMu_mu_ppt'].Fill(P, pt, weight) h['NuTauMu_xvsz_mu'].Fill(z, x, weight) h['NuTauMu_yvsz_mu'].Fill(z, y, weight) for hit in event.TimeDetPoint: if hit: if not hit.GetEnergyLoss() > 0: continue trid = hit.GetTrackID() assert trid > 0 weight = event.MCTrack[trid].GetWeight() x = hit.GetX() y = hit.GetY() z = hit.GetZ() px = hit.GetPx() py = hit.GetPy() pz = hit.GetPz() pt = np.hypot(px, py) P = np.hypot(pz, pt) pid = hit.PdgCode() assert pid not in [12, -12, 14, -14, 16, -16] h['timing_all'].Fill(x, y, weight) if abs(pid) == 13: muon = True muonid = trid h['mu_p'].Fill(P, weight) h['mu_pt'].Fill(pt, weight) h['mu_ppt'].Fill(P, pt, weight) h['timing_mu'].Fill(x, y, weight) for hit in event.muonPoint: if hit: if not hit.GetEnergyLoss() > 0: continue trid = hit.GetTrackID() assert trid > 0 weight = event.MCTrack[trid].GetWeight() x = hit.GetX() y = hit.GetY() px = hit.GetPx() py = hit.GetPy() pz = hit.GetPz() pt = np.hypot(px, py) P = np.hypot(pz, pt) pid = hit.PdgCode() assert pid not in [12, -12, 14, -14, 16, -16] h['muon_tiles_all'].Fill(x, y, weight) h['muon_bars_x_all'].Fill(x, y, weight) h['muon_bars_y_all'].Fill(x, y, weight) if abs(pid) == 13: muon = True muonid = trid h['mu_p'].Fill(P, weight) h['mu_pt'].Fill(pt, weight) h['mu_ppt'].Fill(P, pt, weight) h['muon_tiles_mu'].Fill(x, y, weight) h['muon_bars_y_mu'].Fill(x, y, weight) h['muon_bars_x_mu'].Fill(x, y, weight) for hit in event.vetoPoint: if hit: if not hit.GetEnergyLoss() > 0: continue trid = hit.GetTrackID() assert trid > 0 weight = event.MCTrack[trid].GetWeight() x = hit.GetX() y = hit.GetY() z = hit.GetZ() px = hit.GetPx() py = hit.GetPy() pz = hit.GetPz() pt = np.hypot(px, py) P = np.hypot(pz, pt) pid = hit.PdgCode() detector_ID = hit.GetDetectorID() assert pid not in [12, -12, 14, -14, 16, -16] phi = r.TMath.ATan2(y, x) if 99999 < detector_ID < 999999: h['SBT_Liquid_all'].Fill(z, phi, weight) if abs(pid) == 13: muon = True muonid = trid h['mu_p'].Fill(P, weight) h['mu_pt'].Fill(pt, weight) h['mu_ppt'].Fill(P, pt, weight) h['SBT_Liquid_mu'].Fill(z, phi, weight) continue elif detector_ID > 999999: h['SBT_Plastic_all'].Fill(z, phi, weight) if abs(pid) == 13: muon = True muonid = trid h['mu_p'].Fill(P, weight) h['mu_pt'].Fill(pt, weight) h['mu_ppt'].Fill(P, pt, weight) h['SBT_Plastic_mu'].Fill(z, phi, weight) continue log.warn('Unidentified vetoPoint.') if muon: original_muon = event.MCTrack[muonid] weight = original_muon.GetWeight() h['mu_p_original'].Fill(original_muon.GetP(), weight) h['mu_pt_original'].Fill(original_muon.GetPt(), weight) h['mu_ppt_original'].Fill(original_muon.GetP(), original_muon.GetPt(), weight) # NOTE: muons are counted several times if they create several hits # But the original muon is only counted once. log.info('Event loop done') for key in h: classname = h[key].Class().GetName() if 'TH' in classname or 'TP' in classname: h[key].Write() f.Close()
def GetSeriesYears(dom): #used by the 'add a story arc' option to individually populate the Series Year for each series within the given arc. #series year is required for alot of functionality. series = dom.getElementsByTagName('volume') tempseries = {} serieslist = [] for dm in series: try: totids = len(dm.getElementsByTagName('id')) idc = 0 while (idc < totids): if dm.getElementsByTagName( 'id')[idc].parentNode.nodeName == 'volume': tempseries['ComicID'] = dm.getElementsByTagName( 'id')[idc].firstChild.wholeText idc += 1 except: logger.warn( 'There was a problem retrieving a comicid for a series within the arc. This will have to manually corrected most likely.' ) tempseries['ComicID'] = 'None' tempseries['Series'] = 'None' tempseries['Publisher'] = 'None' try: totnames = len(dm.getElementsByTagName('name')) namesc = 0 while (namesc < totnames): if dm.getElementsByTagName( 'name')[namesc].parentNode.nodeName == 'volume': tempseries['Series'] = dm.getElementsByTagName( 'name')[namesc].firstChild.wholeText elif dm.getElementsByTagName( 'name')[namesc].parentNode.nodeName == 'publisher': tempseries['Publisher'] = dm.getElementsByTagName( 'name')[namesc].firstChild.wholeText namesc += 1 except: logger.warn( 'There was a problem retrieving a Series Name or Publisher for a series within the arc. This will have to manually corrected.' ) try: tempseries['SeriesYear'] = dm.getElementsByTagName( 'start_year')[0].firstChild.wholeText except: logger.warn( 'There was a problem retrieving the start year for a particular series within the story arc.' ) tempseries['SeriesYear'] = '0000' #cause you know, dufus'... if tempseries['SeriesYear'][-1:] == '-': tempseries['SeriesYear'] = tempseries['SeriesYear'][:-1] desdeck = 0 tempseries['Volume'] = 'None' #the description field actually holds the Volume# - so let's grab it try: descchunk = dm.getElementsByTagName( 'description')[0].firstChild.wholeText comic_desc = drophtml(descchunk) desdeck += 1 except: comic_desc = 'None' #sometimes the deck has volume labels try: deckchunk = dm.getElementsByTagName('deck')[0].firstChild.wholeText comic_deck = deckchunk desdeck += 1 except: comic_deck = 'None' while (desdeck > 0): if desdeck == 1: if comic_desc == 'None': comicDes = comic_deck[:30] else: #extract the first 60 characters comicDes = comic_desc[:60].replace('New 52', '') elif desdeck == 2: #extract the characters from the deck comicDes = comic_deck[:30].replace('New 52', '') else: break i = 0 while (i < 2): if 'volume' in comicDes.lower(): #found volume - let's grab it. v_find = comicDes.lower().find('volume') #arbitrarily grab the next 10 chars (6 for volume + 1 for space + 3 for the actual vol #) #increased to 10 to allow for text numbering (+5 max) #sometimes it's volume 5 and ocassionally it's fifth volume. if i == 0: vfind = comicDes[v_find:v_find + 15] #if it's volume 5 format basenums = { 'zero': '0', 'one': '1', 'two': '2', 'three': '3', 'four': '4', 'five': '5', 'six': '6', 'seven': '7', 'eight': '8', 'nine': '9', 'ten': '10', 'i': '1', 'ii': '2', 'iii': '3', 'iv': '4', 'v': '5' } logger.fdebug('volume X format - ' + str(i) + ': ' + vfind) else: vfind = comicDes[: v_find] # if it's fifth volume format basenums = { 'zero': '0', 'first': '1', 'second': '2', 'third': '3', 'fourth': '4', 'fifth': '5', 'sixth': '6', 'seventh': '7', 'eighth': '8', 'nineth': '9', 'tenth': '10', 'i': '1', 'ii': '2', 'iii': '3', 'iv': '4', 'v': '5' } logger.fdebug('X volume format - ' + str(i) + ': ' + vfind) volconv = '' for nums in basenums: if nums in vfind.lower(): sconv = basenums[nums] vfind = re.sub(nums, sconv, vfind.lower()) break #logger.info('volconv: ' + str(volconv)) if i == 0: volthis = vfind.lower().find('volume') volthis = volthis + 6 # add on the actual word to the position so that we can grab the subsequent digit vfind = vfind[volthis:volthis + 4] # grab the next 4 characters ;) elif i == 1: volthis = vfind.lower().find('volume') vfind = vfind[volthis - 4: volthis] # grab the next 4 characters ;) if '(' in vfind: #bracket detected in versioning' vfindit = re.findall('[^()]+', vfind) vfind = vfindit[0] vf = re.findall('[^<>]+', vfind) try: ledigit = re.sub("[^0-9]", "", vf[0]) if ledigit != '': tempseries['Volume'] = ledigit logger.fdebug( "Volume information found! Adding to series record : volume " + tempseries['Volume']) break except: pass i += 1 else: i += 1 if tempseries['Volume'] == 'None': logger.fdebug('tempseries[Volume]:' + str(tempseries['Volume'])) desdeck -= 1 else: break serieslist.append({ "ComicID": tempseries['ComicID'], "ComicName": tempseries['Series'], "SeriesYear": tempseries['SeriesYear'], "Publisher": tempseries['Publisher'], "Volume": tempseries['Volume'] }) return serieslist
def UpdateDates(dom): issues = dom.getElementsByTagName('issue') tempissue = {} issuelist = [] for dm in issues: tempissue['ComicID'] = 'None' tempissue['IssueID'] = 'None' try: totids = len(dm.getElementsByTagName('id')) idc = 0 while (idc < totids): if dm.getElementsByTagName( 'id')[idc].parentNode.nodeName == 'volume': tempissue['ComicID'] = dm.getElementsByTagName( 'id')[idc].firstChild.wholeText if dm.getElementsByTagName( 'id')[idc].parentNode.nodeName == 'issue': tempissue['IssueID'] = dm.getElementsByTagName( 'id')[idc].firstChild.wholeText idc += 1 except: logger.warn( 'There was a problem retrieving a comicid/issueid for the given issue. This will have to manually corrected most likely.' ) tempissue['SeriesTitle'] = 'None' tempissue['IssueTitle'] = 'None' try: totnames = len(dm.getElementsByTagName('name')) namesc = 0 while (namesc < totnames): if dm.getElementsByTagName( 'name')[namesc].parentNode.nodeName == 'issue': tempissue['IssueTitle'] = dm.getElementsByTagName( 'name')[namesc].firstChild.wholeText elif dm.getElementsByTagName( 'name')[namesc].parentNode.nodeName == 'volume': tempissue['SeriesTitle'] = dm.getElementsByTagName( 'name')[namesc].firstChild.wholeText namesc += 1 except: logger.warn( 'There was a problem retrieving the Series Title / Issue Title for a series within the arc. This will have to manually corrected.' ) try: tempissue['CoverDate'] = dm.getElementsByTagName( 'cover_date')[0].firstChild.wholeText except: tempissue['CoverDate'] = '0000-00-00' try: tempissue['StoreDate'] = dm.getElementsByTagName( 'store_date')[0].firstChild.wholeText except: tempissue['StoreDate'] = '0000-00-00' try: tempissue['IssueNumber'] = dm.getElementsByTagName( 'issue_number')[0].firstChild.wholeText except: logger.fdebug( 'No Issue Number available - Trade Paperbacks, Graphic Novels and Compendiums are not supported as of yet.' ) tempissue['IssueNumber'] = 'None' try: tempissue['date_last_updated'] = dm.getElementsByTagName( 'date_last_updated')[0].firstChild.wholeText except: tempissue['date_last_updated'] = '0000-00-00' issuelist.append({ 'ComicID': tempissue['ComicID'], 'IssueID': tempissue['IssueID'], 'SeriesTitle': tempissue['SeriesTitle'], 'IssueTitle': tempissue['IssueTitle'], 'CoverDate': tempissue['CoverDate'], 'StoreDate': tempissue['StoreDate'], 'IssueNumber': tempissue['IssueNumber'], 'Date_Last_Updated': tempissue['date_last_updated'] }) return issuelist
def GetComicInfo(comicid, dom, safechk=None): if safechk is None: #safetycheck when checking comicvine. If it times out, increment the chk on retry attempts up until 5 tries then abort. safechk = 1 elif safechk > 4: logger.error( 'Unable to add / refresh the series due to inablity to retrieve data from ComicVine. You might want to try abit later and/or make sure ComicVine is up.' ) return #comicvine isn't as up-to-date with issue counts.. #so this can get really buggered, really fast. tracks = dom.getElementsByTagName('issue') try: cntit = dom.getElementsByTagName( 'count_of_issues')[0].firstChild.wholeText except: cntit = len(tracks) trackcnt = len(tracks) logger.fdebug("number of issues I counted: " + str(trackcnt)) logger.fdebug("number of issues CV says it has: " + str(cntit)) # if the two don't match, use trackcnt as count_of_issues might be not upto-date for some reason if int(trackcnt) != int(cntit): cntit = trackcnt vari = "yes" else: vari = "no" logger.fdebug("vari is set to: " + str(vari)) #if str(trackcnt) != str(int(cntit)+2): # cntit = int(cntit) + 1 comic = {} comicchoice = [] cntit = int(cntit) #retrieve the first xml tag (<tag>data</tag>) #that the parser finds with name tagName: # to return the parent name of the <name> node : dom.getElementsByTagName('name')[0].parentNode.nodeName # where [0] denotes the number of the name field(s) # where nodeName denotes the parentNode : ComicName = results, publisher = publisher, issues = issue try: names = len(dom.getElementsByTagName('name')) n = 0 comic[ 'ComicPublisher'] = 'Unknown' #set this to a default value here so that it will carry through properly while (n < names): if dom.getElementsByTagName( 'name')[n].parentNode.nodeName == 'results': try: comic['ComicName'] = dom.getElementsByTagName( 'name')[n].firstChild.wholeText comic['ComicName'] = comic['ComicName'].rstrip() except: logger.error( 'There was a problem retrieving the given data from ComicVine. Ensure that www.comicvine.com is accessible AND that you have provided your OWN ComicVine API key.' ) return elif dom.getElementsByTagName( 'name')[n].parentNode.nodeName == 'publisher': try: comic['ComicPublisher'] = dom.getElementsByTagName( 'name')[n].firstChild.wholeText except: comic['ComicPublisher'] = "Unknown" n += 1 except: logger.warn( 'Something went wrong retrieving from ComicVine. Ensure your API is up-to-date and that comicvine is accessible' ) return try: comic['ComicYear'] = dom.getElementsByTagName( 'start_year')[0].firstChild.wholeText except: comic['ComicYear'] = '0000' #safety check, cause you known, dufus'... if comic['ComicYear'][-1:] == '-': comic['ComicYear'] = comic['ComicYear'][:-1] try: comic['ComicURL'] = dom.getElementsByTagName( 'site_detail_url')[trackcnt].firstChild.wholeText except: #this should never be an exception. If it is, it's probably due to CV timing out - so let's sleep for abit then retry. logger.warn( 'Unable to retrieve URL for volume. This is usually due to a timeout to CV, or going over the API. Retrying again in 10s.' ) time.sleep(10) safechk += 1 GetComicInfo(comicid, dom, safechk) desdeck = 0 #the description field actually holds the Volume# - so let's grab it try: descchunk = dom.getElementsByTagName( 'description')[0].firstChild.wholeText comic_desc = drophtml(descchunk) desdeck += 1 except: comic_desc = 'None' #sometimes the deck has volume labels try: deckchunk = dom.getElementsByTagName('deck')[0].firstChild.wholeText comic_deck = deckchunk desdeck += 1 except: comic_deck = 'None' #comic['ComicDescription'] = comic_desc try: comic['Aliases'] = dom.getElementsByTagName( 'aliases')[0].firstChild.wholeText comic['Aliases'] = re.sub('\n', '##', comic['Aliases']).strip() if comic['Aliases'][-2:] == '##': comic['Aliases'] = comic['Aliases'][:-2] #logger.fdebug('Aliases: ' + str(aliases)) except: comic['Aliases'] = 'None' comic['ComicVersion'] = 'None' #noversion' #logger.info('comic_desc:' + comic_desc) #logger.info('comic_deck:' + comic_deck) #logger.info('desdeck: ' + str(desdeck)) #figure out if it's a print / digital edition. comic['Type'] = 'None' if comic_deck != 'None': if any( ['print' in comic_deck.lower(), 'digital' in comic_deck.lower()]): if 'print' in comic_deck.lower(): comic['Type'] = 'Print' elif 'digital' in comic_deck.lower(): comic['Type'] = 'Digital' if comic_desc != 'None' and comic['Type'] == 'None': if 'print' in comic_desc[:60].lower( ) and 'print edition can be found' not in comic_desc.lower(): comic['Type'] = 'Print' elif 'digital' in comic_desc[:60].lower( ) and 'digital edition can be found' not in comic_desc.lower(): comic['Type'] = 'Digital' else: comic['Type'] = 'Print' while (desdeck > 0): if desdeck == 1: if comic_desc == 'None': comicDes = comic_deck[:30] else: #extract the first 60 characters comicDes = comic_desc[:60].replace('New 52', '') elif desdeck == 2: #extract the characters from the deck comicDes = comic_deck[:30].replace('New 52', '') else: break i = 0 while (i < 2): if 'volume' in comicDes.lower(): #found volume - let's grab it. v_find = comicDes.lower().find('volume') #arbitrarily grab the next 10 chars (6 for volume + 1 for space + 3 for the actual vol #) #increased to 10 to allow for text numbering (+5 max) #sometimes it's volume 5 and ocassionally it's fifth volume. if i == 0: vfind = comicDes[v_find:v_find + 15] #if it's volume 5 format basenums = { 'zero': '0', 'one': '1', 'two': '2', 'three': '3', 'four': '4', 'five': '5', 'six': '6', 'seven': '7', 'eight': '8', 'nine': '9', 'ten': '10', 'i': '1', 'ii': '2', 'iii': '3', 'iv': '4', 'v': '5' } logger.fdebug('volume X format - ' + str(i) + ': ' + vfind) else: vfind = comicDes[:v_find] # if it's fifth volume format basenums = { 'zero': '0', 'first': '1', 'second': '2', 'third': '3', 'fourth': '4', 'fifth': '5', 'sixth': '6', 'seventh': '7', 'eighth': '8', 'nineth': '9', 'tenth': '10', 'i': '1', 'ii': '2', 'iii': '3', 'iv': '4', 'v': '5' } logger.fdebug('X volume format - ' + str(i) + ': ' + vfind) volconv = '' for nums in basenums: if nums in vfind.lower(): sconv = basenums[nums] vfind = re.sub(nums, sconv, vfind.lower()) break #logger.info('volconv: ' + str(volconv)) #now we attempt to find the character position after the word 'volume' if i == 0: volthis = vfind.lower().find('volume') volthis = volthis + 6 # add on the actual word to the position so that we can grab the subsequent digit vfind = vfind[volthis:volthis + 4] # grab the next 4 characters ;) elif i == 1: volthis = vfind.lower().find('volume') vfind = vfind[volthis - 4:volthis] # grab the next 4 characters ;) if '(' in vfind: #bracket detected in versioning' vfindit = re.findall('[^()]+', vfind) vfind = vfindit[0] vf = re.findall('[^<>]+', vfind) try: ledigit = re.sub("[^0-9]", "", vf[0]) if ledigit != '': comic['ComicVersion'] = ledigit logger.fdebug( "Volume information found! Adding to series record : volume " + comic['ComicVersion']) break except: pass i += 1 else: i += 1 if comic['ComicVersion'] == 'None': logger.fdebug('comic[ComicVersion]:' + str(comic['ComicVersion'])) desdeck -= 1 else: break if vari == "yes": comic['ComicIssues'] = str(cntit) else: comic['ComicIssues'] = dom.getElementsByTagName( 'count_of_issues')[0].firstChild.wholeText comic['ComicImage'] = dom.getElementsByTagName( 'super_url')[0].firstChild.wholeText comic['ComicImageALT'] = dom.getElementsByTagName( 'small_url')[0].firstChild.wholeText comic['FirstIssueID'] = dom.getElementsByTagName( 'id')[0].firstChild.wholeText # print ("fistIss:" + str(comic['FirstIssueID'])) # comicchoice.append({ # 'ComicName': comic['ComicName'], # 'ComicYear': comic['ComicYear'], # 'Comicid': comicid, # 'ComicURL': comic['ComicURL'], # 'ComicIssues': comic['ComicIssues'], # 'ComicImage': comic['ComicImage'], # 'ComicVolume': ParseVol, # 'ComicPublisher': comic['ComicPublisher'] # }) # comic['comicchoice'] = comicchoice return comic
def pulldetails(comicid, type, issueid=None, offset=1, arclist=None, comicidlist=None): #import easy to use xml parser called minidom: from xml.dom.minidom import parseString if mylar.CONFIG.COMICVINE_API == 'None' or mylar.CONFIG.COMICVINE_API is None: logger.warn( 'You have not specified your own ComicVine API key - it\'s a requirement. Get your own @ http://api.comicvine.com.' ) return else: comicapi = mylar.CONFIG.COMICVINE_API if type == 'comic': if not comicid.startswith('4050-'): comicid = '4050-' + comicid PULLURL = mylar.CVURL + 'volume/' + str(comicid) + '/?api_key=' + str( comicapi ) + '&format=xml&field_list=name,count_of_issues,issues,start_year,site_detail_url,image,publisher,description,first_issue,deck,aliases' elif type == 'issue': if mylar.CONFIG.CV_ONLY: cv_type = 'issues' if arclist is None: searchset = 'filter=volume:' + str( comicid ) + '&field_list=cover_date,description,id,image,issue_number,name,date_last_updated,store_date' else: searchset = 'filter=id:' + ( arclist ) + '&field_list=cover_date,id,issue_number,name,date_last_updated,store_date,volume' else: cv_type = 'volume/' + str(comicid) searchset = 'name,count_of_issues,issues,start_year,site_detail_url,image,publisher,description,store_date' PULLURL = mylar.CVURL + str(cv_type) + '/?api_key=' + str( comicapi) + '&format=xml&' + str(searchset) + '&offset=' + str( offset) elif type == 'firstissue': #this is used ONLY for CV_ONLY PULLURL = mylar.CVURL + 'issues/?api_key=' + str( comicapi) + '&format=xml&filter=id:' + str( issueid) + '&field_list=cover_date' elif type == 'storyarc': PULLURL = mylar.CVURL + 'story_arcs/?api_key=' + str( comicapi) + '&format=xml&filter=name:' + str( issueid) + '&field_list=cover_date' elif type == 'comicyears': PULLURL = mylar.CVURL + 'volumes/?api_key=' + str( comicapi ) + '&format=xml&filter=id:' + str( comicidlist ) + '&field_list=name,id,start_year,publisher,description,deck&offset=' + str( offset) elif type == 'import': PULLURL = mylar.CVURL + 'issues/?api_key=' + str( comicapi ) + '&format=xml&filter=id:' + ( comicidlist ) + '&field_list=cover_date,id,issue_number,name,date_last_updated,store_date,volume' + '&offset=' + str( offset) elif type == 'update_dates': PULLURL = mylar.CVURL + 'issues/?api_key=' + str( comicapi ) + '&format=xml&filter=id:' + ( comicidlist ) + '&field_list=date_last_updated, id, issue_number, store_date, cover_date, name, volume ' + '&offset=' + str( offset) #logger.info('CV.PULLURL: ' + PULLURL) #new CV API restriction - one api request / second. if mylar.CONFIG.CVAPI_RATE is None or mylar.CONFIG.CVAPI_RATE < 2: time.sleep(2) else: time.sleep(mylar.CONFIG.CVAPI_RATE) #download the file: #set payload to None for now... payload = None try: r = requests.get(PULLURL, params=payload, verify=mylar.CONFIG.CV_VERIFY, headers=mylar.CV_HEADERS) except Exception, e: logger.warn('Error fetching data from ComicVine: %s' % (e)) return
def updateNNM(cmdbToNnmIds, nnmConnection, framework): ''' map(string, string), NnmConnection, Framework -> None ''' filePath = _getUpdatedIdsFileName(nnmConnection.serverIp) persistedUpdatedIds = _readUpdatedIdsFromFile(filePath) logger.debug("Have read %s previously updated ID pairs from file" % len(persistedUpdatedIds)) cmdbIdsCount = len(cmdbToNnmIds) # filter out IDs that have not changed _obsoletePersistedIds = {} for cmdbId, nnmId in persistedUpdatedIds.items(): if cmdbToNnmIds.get(cmdbId) == nnmId: del cmdbToNnmIds[cmdbId] else: _obsoletePersistedIds[cmdbId] = nnmId # filter out persisted IDs that are obsolete for obsoleteId in _obsoletePersistedIds.keys(): del persistedUpdatedIds[obsoleteId] stub = _createNnmStub(nnmConnection) filteredCmdbIdsCount = len(cmdbToNnmIds) logger.debug("Custom Attribute UCMDB_ID on %d NNM nodes in the NNM topology will be updated with UCMDB IDs" % filteredCmdbIdsCount) if filteredCmdbIdsCount > 3000: logger.warn("Getting ready to update Custom Attribute UCMDB_ID on %d NNM nodes in NNM" % filteredCmdbIdsCount) logger.warn("This process may take a while since the UCMDB_ID custom attribute in NNM can only be updated one node at a time. Check probeMgr-patternsDebug.log for status update.") updatedIds = {} counter = 0 for cmdbId, nnmId in cmdbToNnmIds.items(): nodes = _getNnmNodeById(stub, nnmId) if not nodes: logger.warn("Node with NNM ID %s doesn't exists in NNMi" % nnmId) framework.reportWarning("One of nodes was not pushed since it doesn't exist in NNMi") continue try: _updateNnmNode(nnmConnection, stub, cmdbId, nnmId, framework) except UnsupportedNnmMethodException: nnmConnection.version = 8 logger.debug('Detected NNMi version 8.x') #retry the update _updateNnmNode(nnmConnection, stub, cmdbId, nnmId, framework) else: if nnmConnection.version == -1: nnmConnection.version = 9 logger.debug('Detected NNMi version 9.x') logger.debug("(%d) Updated NNM ID: %s with UCMDB ID value %s" % (counter, nnmId, cmdbId)) counter += 1 updatedIds[cmdbId] = nnmId _saveUpdatedIdsToFile(filePath, persistedUpdatedIds, updatedIds) logger.debug("Finished updating %d (out of %d) nodes in NNM" % (filteredCmdbIdsCount, cmdbIdsCount))
def discover(self, connectionContext): cloud = vcloud.Vcloud() cloud.ipAddress = connectionContext.ipAddress cloud.urlString = connectionContext.urlString client = connectionContext.client agent = client.getAgent() vcloudClient = agent.getVcloudClient() organizationsByName = self.getOrganizations(vcloudClient) cloud.organizationsByName = organizationsByName for organization in organizationsByName.values(): logger.debug(organization) organizationInstance = organization.getInstance() vdcByName = self.getVdcForOrganization(organizationInstance, vcloudClient) organization.vdcByName = vdcByName for vdc in vdcByName.values(): logger.debug(vdc) # additional references for resolving vdcInstance = vdc.getInstance() vdcReference = vdcInstance.getReference() vdcHref = vdcReference.getHref() organization._vdcByHref[vdcHref] = vdc vappByName = self.getVappsForVdc(vdcInstance, vcloudClient) vdc.vappsByName = vappByName for vapp in vappByName.values(): logger.debug(vapp) vappInstance = vapp.getInstance() vmsByName = self.getVmsForVapp(vappInstance, vcloudClient) vapp.vmsByName = vmsByName for vm in vmsByName.values(): vm._findHostKeyAndIps() catalogsByName = self.getCatalogsByNameForOrganization( organizationInstance, vcloudClient) organization.catalogsByName = catalogsByName for catalog in catalogsByName.values(): logger.debug(catalog) catalogInstance = catalog.getInstance() catalogEntriesByType = self.getCatalogEntriesByTypeForCatalog( catalogInstance, vcloudClient) mediaByName = catalogEntriesByType[VcloudReferenceType.MEDIA] vappTemplatesByName = catalogEntriesByType[ VcloudReferenceType.VAPP_TEMPLATE] catalog.mediaByName = mediaByName catalog.vappTemplatesByName = vappTemplatesByName vcloudAdmin = self.getAdmin(vcloudClient) if vcloudAdmin is not None: self.getVcloudDirectorVersion(cloud, vcloudAdmin) systemOrganization = self.getSystemOrganization( vcloudClient, vcloudAdmin) cloud.systemOrganization = systemOrganization if Configuration.SKIP_DUPLICATING_SYSTEM_ORG: self._removeSystemOrganizationDuplicates(cloud) adminOrganizationsByName = self.getAdminOrganizations( vcloudClient, vcloudAdmin) for adminOrganizationName, adminOrganization in adminOrganizationsByName.items( ): organization = organizationsByName.get(adminOrganizationName) if organization is not None: organization.setAdminOrganization(adminOrganization) elif cloud.systemOrganization is not None and adminOrganizationName == cloud.systemOrganization.getName( ): cloud.systemOrganization.setAdminOrganization( adminOrganization) else: logger.warn( "No regular organization for admin organization '%s'" % adminOrganizationName) providerVdcByName = self.getProviderVdc(vcloudClient, vcloudAdmin) cloud.providerVdcByName = providerVdcByName if not providerVdcByName: logger.debug("Provider vDC not available") for providerVdc in providerVdcByName.values(): logger.debug(providerVdc) providerVdcInstance = providerVdc.getInstance() adminVdcByName = self.getAdminVdcForProviderVdc( providerVdcInstance, vcloudClient) for adminVdcName, adminVdc in adminVdcByName.items(): for organization in cloud.organizationsByName.values(): vdc = organization.vdcByName.get(adminVdcName) if vdc is not None: vdc.setAdminVdc(adminVdc) vdc.providerVdcName = providerVdc.getName() else: logger.debug("vCloud admin not available") self.discoverGlobalVcloudSettings(vcloudClient, cloud) return cloud
def isHexadecimalName(dataName): if dataName and re.match(r"^[\dabcdefABCDEF]+$", dataName.strip()): logger.warn('Skipping Hex Software: ' + dataName) return 1
def _report_db_host(topology, resolver): '@types: _Topology, dns_resolver.Resolver -> osh, list[osh]' osh, oshs = None, [] if topology.db_info and resolver: hostname = topology.db_info.hostname logger.info("Resolve database address: %s" % hostname) try: try: ips = resolver.resolve_ips(hostname) except dns_resolver.ResolveException, e: logger.warn("Failed to resolve. %s" % e) else: osh, oshs = _report_host(sap.Address(hostname, ips)) except ValueError, e: logger.warn("Failed to resolve %s" % e) return osh, oshs def _report_application_ip(shell, topology, resolver): '@types: _Topology, dns_resolver.Resolver -> string' if topology.inst and resolver: hostname = topology.inst.getHostname() logger.info("Resolve host address from profile: %s" % hostname) if not hostname: cmd = "hostname" output = shell.execCmd(cmd) if not shell.getLastCmdReturnCode(): hostname = output logger.info("Resolve host address from host: %s" % hostname)
def processor(self, nzbinfo): nzbid = nzbinfo['NZBID'] try: logger.fdebug( 'Now checking the active queue of nzbget for the download') queueinfo = self.server.listgroups() except Expection as e: logger.warn( 'Error attempting to retrieve active queue listing: %s' % e) return {'status': False} else: logger.fdebug('valid queue result returned. Analyzing...') queuedl = [qu for qu in queueinfo if qu['NZBID'] == nzbid] if len(queuedl) == 0: logger.warn( 'Unable to locate item in active queue. Could it be finished already ?' ) return {'status': False} stat = False while stat is False: time.sleep(10) queueinfo = self.server.listgroups() queuedl = [qu for qu in queueinfo if qu['NZBID'] == nzbid] if len(queuedl) == 0: logger.fdebug( 'Item is no longer in active queue. It should be finished by my calculations' ) stat = True else: logger.fdebug('status: %s' % queuedl[0]['Status']) logger.fdebug('name: %s' % queuedl[0]['NZBName']) logger.fdebug('FileSize: %sMB' % queuedl[0]['FileSizeMB']) logger.fdebug('Download Left: %sMB' % queuedl[0]['RemainingSizeMB']) logger.fdebug('health: %s' % (queuedl[0]['Health'] / 10)) logger.fdebug('destination: %s' % queuedl[0]['DestDir']) logger.fdebug('File has now downloaded!') time.sleep( 5 ) #wait some seconds so shit can get written to history properly history = self.server.history() found = False hq = [ hs for hs in history if hs['NZBID'] == nzbid and 'SUCCESS' in hs['Status'] ] if len(hq) > 0: logger.fdebug( 'found matching completed item in history. Job has a status of %s' % hq[0]['Status']) if hq[0]['DownloadedSizeMB'] == hq[0]['FileSizeMB']: logger.fdebug('%s has final file size of %sMB' % (hq[0]['Name'], hq[0]['DownloadedSizeMB'])) if os.path.isdir(hq[0]['DestDir']): logger.fdebug('location found @ %s' % hq[0]['DestDir']) return { 'status': True, 'name': re.sub('.nzb', '', hq[0]['NZBName']).strip(), 'location': hq[0]['DestDir'], 'failed': False } else: logger.warn( 'no file found where it should be @ %s - is there another script that moves things after completion ?' % hq[0]['DestDir']) return {'status': False} else: logger.warn('Could not find completed item in history') return {'status': False}
def DiscoveryMain(Framework): OSHVResult = ObjectStateHolderVector() ms_domain_name = Framework.getDestinationAttribute('ms_domain_name') if not ms_domain_name: ms_domain_name = 'NULL' try: netUtil = MsNetworkUtil() hostsOutput = netUtil.doNetServerEnum('NULL', SV_TYPE_SERVER, ms_domain_name) if hostsOutput != None: discoverUnknownIPs = 1 try: strDiscoverUnknownIPs = Framework.getParameter( 'discoverUnknownIPs') discoverUnknownIPs = Boolean.parseBoolean( strDiscoverUnknownIPs) except: pass oshMsDomain = ObjectStateHolder('msdomain') oshMsDomain.setStringAttribute('data_name', ms_domain_name) alreadyDiscoveredIps = HashMap() for hostInfo in hostsOutput: hostType = Long(hostInfo[1]).longValue() hostName = (str(hostInfo[0])).lower() try: ip = InetAddress.getByName(hostInfo[0]).getHostAddress() if netutils.isLocalIp(ip): continue cachedHostName = alreadyDiscoveredIps.get(ip) if cachedHostName != None: logger.debug( 'IP ', ip, ' already reported for host ' + cachedHostName, ' current host ', hostName, ' - skipping') continue else: logger.debug('Discovered IP ' + ip + ' for host ' + hostName) alreadyDiscoveredIps.put(ip, hostName) ipDomain = DomainScopeManager.getDomainByIp(ip) if not discoverUnknownIPs and ipDomain == 'unknown': logger.debug( 'ip: ' + ip + ' is out of probe range and will be excluded') continue if SV_TYPE_CLUSTER_NT & hostType: logger.debug( 'Not reporting the entry %s because it is a Cluster' % hostName) continue hostOsType = 'nt' if SV_TYPE_SERVER_UNIX & hostType: hostOsType = 'unix' oshHost = modeling.createHostOSH(ip, hostOsType) oshHost.setStringAttribute("host_hostname", hostName) OSHVResult.add(oshHost) link = modeling.createLinkOSH('member', oshMsDomain, oshHost) OSHVResult.add(link) ipOSH = modeling.createIpOSH(ip) OSHVResult.add(ipOSH) contained = modeling.createLinkOSH('contained', oshHost, ipOSH) OSHVResult.add(contained) except: errorMsg = str(sys.exc_info()[1]).strip() logger.warn('Failed to resolve host ', hostInfo[0], ' : ', errorMsg) else: message = 'Failed to discover hosts on MS Domain' logger.warn(message) logger.reportWarning(message) except: errorMsg = str(sys.exc_info()[1]).strip() logger.errorException('Failed to discovery MS Domains') errorMessage = errormessages.makeErrorMessage( "msdomain", errorMsg, errormessages.ERROR_FAILED_DISCOVERING_MSDOMAIN_HOSTS) errobj = errorobject.createError( errorcodes.FAILED_DISCOVERIING_MSDOMAIN_HOST, ["msdomain", errorMsg], errorMessage) logger.reportErrorObject(errobj) return OSHVResult
def parseTransportAdaptersForNode(node, scconfOutput): adaptersList = [] matcher = re.search( r"\(%s\) Node transport adapters:([^\n]+)" % re.escape(node.name), scconfOutput) if matcher: adaptersListStr = matcher.group(1) and matcher.group(1).strip() if adaptersListStr: adaptersList = re.split(r"\s+", adaptersListStr) adaptersList = [adapter for adapter in adaptersList if adapter] if adaptersList: for adapterName in adaptersList: adapterStatus = None tuple = (re.escape(node.name), re.escape(adapterName)) matcher = re.search(r"\(%s:%s\) Adapter enabled:([^\n]+)" % tuple, scconfOutput) if matcher: adapterStatus = matcher.group(1) and matcher.group(1).strip() if adapterStatus != 'yes': logger.debug("Skipping disabled transport adapter '%s'" % adapterName) continue logger.debug("Found transport adapter '%s'" % adapterName) transportAdapter = TransportAdapter(adapterName) properties = {} results = re.findall( r"\(%s:%s\) Adapter property:([^\n]+)" % tuple, scconfOutput) if results: for row in results: elements = re.split(r"=", row, 1) if len(elements) == 2: propertyName = elements[0] and elements[0].strip() propertyValue = elements[1] and elements[1].strip() if propertyName and propertyValue: properties[propertyName] = propertyValue ip = properties.get('ip_address') if ip and netutils.isValidIp(ip): transportAdapter.ip = ip logger.debug("Adapter's private IP is '%s'" % ip) else: logger.warn( "Could not find private IP for transport adapter '%s' on node '%s'" % (adapterName, node.name)) netmask = properties.get('netmask') if netmask: try: transportAdapter.netmask = netutils.parseNetMask(netmask) except: logger.warn("Failed parsing netmask: %s" % netmask) node.transportAdaptersByName[adapterName] = transportAdapter else: logger.warn("No transport adapters found for node '%s'" % node.name)
def run(): from websocket import create_connection if plexpy.CONFIG.PMS_SSL and plexpy.CONFIG.PMS_URL[:5] == 'https': uri = plexpy.CONFIG.PMS_URL.replace( 'https://', 'wss://') + '/:/websockets/notifications' secure = 'secure ' else: uri = 'ws://%s:%s/:/websockets/notifications' % ( plexpy.CONFIG.PMS_IP, plexpy.CONFIG.PMS_PORT) secure = '' # Set authentication token (if one is available) if plexpy.CONFIG.PMS_TOKEN: header = ["X-Plex-Token: %s" % plexpy.CONFIG.PMS_TOKEN] else: header = [] global ws_shutdown ws_shutdown = False reconnects = 0 # Try an open the websocket connection while not plexpy.WS_CONNECTED and reconnects < plexpy.CONFIG.WEBSOCKET_CONNECTION_ATTEMPTS: if reconnects == 0: logger.info(u"Tautulli WebSocket :: Opening %swebsocket." % secure) reconnects += 1 # Sleep 5 between connection attempts if reconnects > 1: time.sleep(plexpy.CONFIG.WEBSOCKET_CONNECTION_TIMEOUT) logger.info(u"Tautulli WebSocket :: Connection attempt %s." % str(reconnects)) try: plexpy.WEBSOCKET = create_connection(uri, header=header) logger.info(u"Tautulli WebSocket :: Ready") plexpy.WS_CONNECTED = True except (websocket.WebSocketException, IOError, Exception) as e: logger.error("Tautulli WebSocket :: %s." % e) if plexpy.WS_CONNECTED: on_connect() while plexpy.WS_CONNECTED: try: process(*receive(plexpy.WEBSOCKET)) # successfully received data, reset reconnects counter reconnects = 0 except websocket.WebSocketConnectionClosedException: if ws_shutdown: break if reconnects == 0: logger.warn(u"Tautulli WebSocket :: Connection has closed.") if not plexpy.CONFIG.PMS_IS_CLOUD and reconnects < plexpy.CONFIG.WEBSOCKET_CONNECTION_ATTEMPTS: reconnects += 1 # Sleep 5 between connection attempts if reconnects > 1: time.sleep(plexpy.CONFIG.WEBSOCKET_CONNECTION_TIMEOUT) logger.warn(u"Tautulli WebSocket :: Reconnection attempt %s." % str(reconnects)) try: plexpy.WEBSOCKET = create_connection(uri, header=header) logger.info(u"Tautulli WebSocket :: Ready") plexpy.WS_CONNECTED = True except (websocket.WebSocketException, IOError, Exception) as e: logger.error("Tautulli WebSocket :: %s." % e) else: close() break except (websocket.WebSocketException, Exception) as e: if ws_shutdown: break logger.error("Tautulli WebSocket :: %s." % e) close() break if not plexpy.WS_CONNECTED and not ws_shutdown: on_disconnect() logger.debug(u"Tautulli WebSocket :: Leaving thread.")
def render(self): logger.warn('Render not defined for %s' % self)
def discover(self, include_cloud=True, all_servers=False): """ Query plex for all servers online. Returns the ones you own in a selectize format """ # Try to discover localhost server local_machine_identifier = None request_handler = http_handler.HTTPHandler(urls='http://127.0.0.1:32400', timeout=1, ssl_verify=False, silent=True) request = request_handler.make_request(uri='/identity', request_type='GET', output_format='xml') if request: xml_head = request.getElementsByTagName('MediaContainer')[0] local_machine_identifier = xml_head.getAttribute('machineIdentifier') local_server = {'httpsRequired': '0', 'clientIdentifier': local_machine_identifier, 'label': 'Local', 'ip': '127.0.0.1', 'port': '32400', 'uri': 'http://127.0.0.1:32400', 'local': '1', 'value': '127.0.0.1:32400', 'is_cloud': False } servers = self.get_plextv_resources(include_https=True, output_format='xml') clean_servers = [] try: xml_head = servers.getElementsByTagName('MediaContainer') except Exception as e: logger.warn("Tautulli PlexTV :: Failed to get servers from plex: %s." % e) return [] for a in xml_head: if a.getAttribute('size'): if a.getAttribute('size') == '0': return [] if a.getElementsByTagName('Device'): devices = a.getElementsByTagName('Device') for d in devices: if helpers.get_xml_attr(d, 'presence') == '1' and \ helpers.get_xml_attr(d, 'owned') == '1' and \ helpers.get_xml_attr(d, 'provides') == 'server': is_cloud = (helpers.get_xml_attr(d, 'platform').lower() == 'cloud') if not include_cloud and is_cloud: continue connections = d.getElementsByTagName('Connection') for c in connections: if not all_servers: # If this is a remote server don't show any local IPs. if helpers.get_xml_attr(d, 'publicAddressMatches') == '0' and \ helpers.get_xml_attr(c, 'local') == '1': continue # If this is a local server don't show any remote IPs. if helpers.get_xml_attr(d, 'publicAddressMatches') == '1' and \ helpers.get_xml_attr(c, 'local') == '0': continue if helpers.get_xml_attr(d, 'clientIdentifier') == local_machine_identifier: local_server['httpsRequired'] = helpers.get_xml_attr(d, 'httpsRequired') local_server['label'] = helpers.get_xml_attr(d, 'name') clean_servers.append(local_server) local_machine_identifier = None server = {'httpsRequired': '1' if is_cloud else helpers.get_xml_attr(d, 'httpsRequired'), 'clientIdentifier': helpers.get_xml_attr(d, 'clientIdentifier'), 'label': helpers.get_xml_attr(d, 'name'), 'ip': helpers.get_xml_attr(c, 'address'), 'port': helpers.get_xml_attr(c, 'port'), 'uri': helpers.get_xml_attr(c, 'uri'), 'local': helpers.get_xml_attr(c, 'local'), 'value': helpers.get_xml_attr(c, 'address') + ':' + helpers.get_xml_attr(c, 'port'), 'is_cloud': is_cloud } clean_servers.append(server) if local_machine_identifier: clean_servers.append(local_server) clean_servers.sort(key=lambda s: (s['label'], -int(s['local']), s['ip'])) return clean_servers