def _execute( cmd, callback = None, user_data = None ):
logger.debug("Call command \"%s\"" %cmd, traceDepth = 1)
ret_val = 0
if callback is None:
ret_val = os.system( cmd )
else:
pipe = os.popen( cmd, 'r' )
while True:
line = temp_failure_retry( pipe.readline )
if not line:
break
callback( line.strip(), user_data )
ret_val = pipe.close()
if ret_val is None:
ret_val = 0
if ret_val != 0:
logger.warning("Command \"%s\" returns %s"
%(cmd, ret_val),
traceDepth = 1)
else:
logger.debug("Command \"%s...\" returns %s"
%(cmd[:min(16, len(cmd))], ret_val),
traceDepth = 1)
return ret_val
def addCisWithDependency(self, osh):
if self.__vector.contains(osh) or osh not in self.needRelationshipCIs: # already in or doesn't need dependency
return True
deps = self.needRelationshipCIs[osh].split(',')
allFound = True
for dep in deps:
relationship, targetCIType = dep.split(':')
singleFound = False
if osh in self.linksMap:
links = self.linksMap[osh]
logger.debug("Get link of target ci", osh)
for link in links:
if link.getObjectClass() == relationship:
end1 = link.getAttributeValue('link_end1')
end2 = link.getAttributeValue('link_end2')
that = end1 if end2 == osh else end2
if that.getObjectClass() == targetCIType:
singleFound = self.addAllDependencies(that)
if singleFound:
break
if not singleFound:
allFound = False
break
if allFound:
self.__vector.add(osh)
else:
for link in self.linksMap[osh]:
self.__vector.remove(link)
return allFound
def create_https_certificates(ssl_cert, ssl_key):
"""
Create a self-signed HTTPS certificate and store in it in
'ssl_cert' and 'ssl_key'. Method assumes pyOpenSSL is installed.
This code is stolen from SickBeard (http://github.com/midgetspy/Sick-Beard).
"""
from OpenSSL import crypto
from certgen import createKeyPair, createSelfSignedCertificate, TYPE_RSA
serial = int(time.time())
domains = ['DNS:' + d.strip() for d in plexpy.CONFIG.HTTPS_DOMAIN.split(',') if d]
ips = ['IP:' + d.strip() for d in plexpy.CONFIG.HTTPS_IP.split(',') if d]
altNames = ','.join(domains + ips)
# Create the self-signed PlexPy certificate
logger.debug(u"Generating self-signed SSL certificate.")
pkey = createKeyPair(TYPE_RSA, 2048)
cert = createSelfSignedCertificate(("PlexPy", pkey), serial, (0, 60 * 60 * 24 * 365 * 10), altNames) # ten years
# Save the key and certificate to disk
try:
with open(ssl_cert, "w") as fp:
fp.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert))
with open(ssl_key, "w") as fp:
fp.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey))
except IOError as e:
logger.error("Error creating SSL key and certificate: %s", e)
return False
return True
def inhibitSuspend( app_id = sys.argv[0],
toplevel_xid = None,
reason = 'take snapshot',
flags = INHIBIT_SUSPENDING | INHIBIT_IDLE):
"""
Prevent machine to go to suspend or hibernate.
Returns the inhibit cookie which is used to end the inhibitor.
"""
if not app_id:
app_id = 'backintime'
if not toplevel_xid:
toplevel_xid = 0
for dbus_props in INHIBIT_DBUS:
try:
#connect directly to the socket instead of dbus.SessionBus because
#the dbus.SessionBus was initiated before we loaded the environ
#variables and might not work
if 'DBUS_SESSION_BUS_ADDRESS' in os.environ:
bus = dbus.bus.BusConnection(os.environ['DBUS_SESSION_BUS_ADDRESS'])
else:
bus = dbus.SessionBus()
interface = bus.get_object(dbus_props['service'], dbus_props['objectPath'])
proxy = interface.get_dbus_method(dbus_props['methodSet'], dbus_props['interface'])
cookie = proxy(*[ (app_id, dbus.UInt32(toplevel_xid), reason, dbus.UInt32(flags))[i] for i in dbus_props['arguments'] ])
logger.info('Inhibit Suspend started. Reason: %s' % reason)
return (cookie, bus, dbus_props)
except dbus.exceptions.DBusException:
pass
if isRoot():
logger.debug("Inhibit Suspend failed because BIT was started as root.")
return
logger.warning('Inhibit Suspend failed.')
def cache_image(url, image=None):
"""
Saves an image to the cache directory.
If no image is provided, tries to return the image from the cache directory.
"""
# Create image directory if it doesn't exist
imgdir = os.path.join(plexpy.CONFIG.CACHE_DIR, 'images/')
if not os.path.exists(imgdir):
logger.debug(u"PlexPy Helpers :: Creating image cache directory at %s" % imgdir)
os.makedirs(imgdir)
# Create a hash of the url to use as the filename
imghash = hashlib.md5(url).hexdigest()
imagefile = os.path.join(imgdir, imghash)
# If an image is provided, save it to the cache directory
if image:
try:
with open(imagefile, 'wb') as cache_file:
cache_file.write(image)
except IOError as e:
logger.error(u"PlexPy Helpers :: Failed to cache image %s: %s" % (imagefile, e))
# Try to return the image from the cache directory
if os.path.isfile(imagefile):
imagetype = 'image/' + imghdr.what(os.path.abspath(imagefile))
else:
imagefile = None
imagetype = 'image/jpeg'
return imagefile, imagetype
def discoverPlans(self,oshv,sqlServerId,dbs):
logger.debug("going to get jobs and plans")
if self.discoveryOptions and self.discoveryOptions.discoverSqlJob:
jobById=self.getSqlJobs(oshv, sqlServerId)
else:
jobById=HashMap()
rs = self.connection.getTable(self.plansQuery)
plans = HashMap()
while(rs.next()):
name = rs.getString('plan_name')
id = rs.getString('plan_id')
osh = ObjectStateHolder('sqlservermaintenanceplan')
osh.setAttribute(Queries.DATA_NAME,name)
osh.setAttribute('planId',id)
osh.setContainer(sqlServerId)
oshv.add(osh)
if self.discoveryOptions and self.discoveryOptions.discoverDbUser:
owner = rs.getString('owner')
# Some plans may not have an owner so we need to check
if owner:
user = ObjectStateHolder('dbuser')
user.setAttribute(Queries.DATA_NAME,owner)
user.setContainer(sqlServerId)
oshv.add(user)
oshv.add(modeling.createLinkOSH('owner',user,osh))
plans.put(name,osh)
rs.close()
logger.debug("got plans: ", plans.keySet().toString())
self.discoverPlanJobs(oshv,sqlServerId,plans,jobById)
self.discoverPlanDbs(oshv,plans,dbs)
def collectData(self,hostId,sqlServerId, discoverConfigs = 1):
self.connection.open()
oshv = ObjectStateHolderVector()
try:
oshv.add(self.getServerProperties(sqlServerId,hostId))
dbMap = self.getDatabases(sqlServerId)
#get the databases
oshv.addAll(self.sqlDataBaseProps.getDatabases(dbMap,hostId,discoverConfigs))
oshv.addAll(self.sqlDataBaseProps.getStoredProcedures(dbMap))
#get the server configuration:
logger.debug('discovering configs')
try:
oshv.add(self.sqlServerConfig.getServerConfiguration(sqlServerId))
oshv.add(self.sqlServerConfig.getServerStartup(sqlServerId))
self.sqlServerConfig.discoverPlans(oshv,sqlServerId,dbMap)
except:
logger.debugException(hostId.toString())
if self.discoveryOptions and self.discoveryOptions.discoverDbUser:
users = self.getDbUsers(sqlServerId)
Util.addFromMap(users,oshv)
else:
users = LinkedHashMap()
oshv.addAll(self.getProcesses(hostId,sqlServerId,dbMap,users))
oshv.addAll(self.clusterConfiguration.collectData(sqlServerId))
#db configuration:
oshv.addAll(self.getDbConf(dbMap,hostId,users))
logger.debug("sql db result for hostid:"+hostId.toString())
except:
logger.debugException(hostId.toString())
self.connection.close()
return oshv
def connectToRemoteNode(Framework):
if AgentUtils.isMigrateNeeded(Framework):
#setting connected client identifier
#using host name since uduid is stored in agent options and on old and new ddmi agent their location is different
logger.debug('Connected using uda.')
client = Framework.getConnectedClient()
sysInfo = client.getSysInfo()
hostName = sysInfo.getProperty('computerName')
Framework.setProperty(InventoryUtils.UD_HOSTNAME, hostName)
AgentUtils.setUdAgentProtocolForMigration(Framework, client.getCredentialId())
logger.debug('Migrate is going to be performed')
if client.hasShell():
logger.debug('The connected Agent already supports shell, assume it is a non-native agent.')
reason = 'The connected Agent already supports shell,it may be a non-native agent.'
Framework.setProperty(InventoryUtils.generateSkipStep('Install Non-Native UD Agent'), reason)
#Framework.setProperty(InventoryUtils.generateSkipStep('Check Non-Native Agent Installed'), reason)
platform = Framework.getProperty(InventoryUtils.STATE_PROPERTY_PLATFORM)
if platform == 'windows':
# In windows, it is native already if it has shell.
logger.debug('This is windows, it must be native agent.')
Framework.setProperty(AgentUtils.DOWNLOAD_MIGRATE_LOG_FILE, '')
reason = 'Native installation is used for Windows platform.'
Framework.setProperty(InventoryUtils.generateSkipStep('Init Update from Non-Native to Native'), reason)
Framework.setProperty(InventoryUtils.generateSkipStep('Install Native UD Agent'), reason)
else:
logger.debug('The connected client does NOT support the shell capability. This is DDMi agent!')
Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
def _mount(self):
"""
mount the service
"""
sshfs = [self.mountproc] + self.ssh_options
if not self.cipher == 'default':
sshfs.extend(['-o', 'Ciphers=%s' % self.cipher])
sshfs.extend(['-o', 'idmap=user',
'-o', 'cache_dir_timeout=2'])
# use read only mount if requested
if self.read_only:
sshfs.extend(['-o', 'ro'])
sshfs.extend([self.user_host_path, self.mountpoint])
#bugfix: sshfs doesn't mount if locale in LC_ALL is not available on remote host
#LANG or other envirnoment variable are no problem.
env = os.environ.copy()
if 'LC_ALL' in list(env.keys()):
env['LC_ALL'] = 'C'
logger.debug('Call mount command: %s'
%' '.join(sshfs),
self)
try:
subprocess.check_call(sshfs, env = env)
except subprocess.CalledProcessError:
raise MountException( _('Can\'t mount %s') % ' '.join(sshfs))
def on_stop(self, force_stop=False):
if self.is_valid_session():
logger.debug(u"PlexPy ActivityHandler :: Session %s has stopped." % str(self.get_session_key()))
# Set the session last_paused timestamp
ap = activity_processor.ActivityProcessor()
ap.set_session_last_paused(session_key=self.get_session_key(), timestamp=None)
# Update the session state and viewOffset
# Set force_stop to true to disable the state set
if not force_stop:
ap.set_session_state(session_key=self.get_session_key(),
state=self.timeline['state'],
view_offset=self.timeline['viewOffset'],
stopped=int(time.time()))
# Retrieve the session data from our temp table
db_session = ap.get_session_by_key(session_key=self.get_session_key())
# Check if any notification agents have notifications enabled
if any(d['on_stop'] for d in notifiers.available_notification_agents()):
# Fire off notifications
threading.Thread(target=notification_handler.notify,
kwargs=dict(stream_data=db_session, notify_action='stop')).start()
# Write it to the history table
monitor_proc = activity_processor.ActivityProcessor()
monitor_proc.write_session_history(session=db_session)
# Remove the session from our temp session table
logger.debug(u"PlexPy ActivityHandler :: Removing session %s from session queue" % str(self.get_session_key()))
ap.delete_session(session_key=self.get_session_key())
def StepMain(Framework):
# if we have shell credentials and we are able to connect with them then connect otherwise we should connect with agent
ip = Framework.getDestinationAttribute('ip_address')
domain = Framework.getDestinationAttribute('ip_domain')
codepage = Framework.getCodePage()
allShellProtocols = []
allShellCredentials = []
allShellIps = []
allShellCodePages = []
protocols = netutils.getAvailableProtocols(Framework, ClientsConsts.DDM_AGENT_PROTOCOL_NAME, ip, domain)
for protocol in protocols:
allShellProtocols.append(ClientsConsts.DDM_AGENT_PROTOCOL_NAME)
allShellCredentials.append(protocol)
allShellIps.append(ip)
allShellCodePages.append(codepage)
logger.debug('Will going to attempt to connect in this order: ', allShellCredentials)
Framework.setProperty(InventoryUtils.STATE_PROPERTY_CONNECTION_PROTOCOLS, allShellProtocols)
Framework.setProperty(InventoryUtils.STATE_PROPERTY_CONNECTION_CREDENIALS, allShellCredentials)
Framework.setProperty(InventoryUtils.STATE_PROPERTY_CONNECTION_IPS, allShellIps)
Framework.setProperty(InventoryUtils.STATE_PROPERTY_CONNECTION_CODEPAGES, allShellCodePages)
InventoryUtils.executeStep(Framework, connectToRemoteNode, InventoryUtils.STEP_REQUIRES_CONNECTION, InventoryUtils.STEP_DOESNOT_REQUIRES_LOCK)
def accel_calibrate_reference():
"""run accelcal on reference board"""
logger.info("STARTING REFERENCE ACCEL CALIBRATION")
conn = connection.Connection(ref_only=True)
logger.info("Turning safety off")
rotate.set_rotation(conn, "level", wait=False)
util.safety_off(conn.refmav)
conn.ref.send("accelcal\n")
for rotation in ["level", "left", "right", "up", "down", "back"]:
try:
conn.ref.expect("Place vehicle")
conn.ref.expect("and press any key")
except Exception as ex:
util.failure("Failed to get place vehicle message for %s" % rotation)
logger.debug("Rotating %s" % rotation)
attitude = rotate.set_rotation(conn, rotation, wait=False)
time.sleep(13)
conn.ref.send("\n")
i = conn.ref.expect(["Calibration successful", "Calibration FAILED"])
if i != 0:
util.failure("Accel calibration failed at %s" % time.ctime())
logger.info("Calibration successful")
rotate.set_rotation(conn, "level", wait=False)
util.param_set(conn.ref, "AHRS_TRIM_X", 0)
util.param_set(conn.ref, "AHRS_TRIM_Y", 0)
util.discard_messages(conn.refmav)
util.wait_heartbeat(conn.refmav)
def on_created(self):
if self.is_item():
logger.debug(u"PlexPy TimelineHandler :: Library item %s has been added to Plex." % str(self.get_rating_key()))
# Fire off notifications
threading.Thread(target=notification_handler.notify_timeline,
kwargs=dict(timeline_data=self.get_metadata(), notify_action='created')).start()
def getMAC(shell, int_name):
cmdResult = None
rawCmdResult = None
mac = None
entstat = None
try:
entstat_command = concatenate('entstat ', int_name)
logger.debug(concatenate(' Executing command: ', entstat_command))
entstat = shell.execCmd(entstat_command)
if entstat != None:
m = re.search('Device Type: (.+)', entstat)
description = None
if(m):
description = m.group(1).strip()
m = re.search('Hardware Address: ([0-9a-f:]{17})', entstat)
rawMac = None
if(m):
rawMac = m.group(1)
mac = netutils.parseMac(rawMac)
except:
msg = " Failed getting MAC address for interface '%s'" % int_name
errobj = errorobject.createError(errorcodes.FAILED_GETTING_INFORMATION, None, msg)
logger.reportWarningObject(errobj)
logger.debug(msg)
return None
return mac
def DiscoveryMain(Framework):
OSHVResult = ObjectStateHolderVector()
logger.info('Starting HACMP Applications')
hostIP = Framework.getDestinationAttribute('ip_address')
logger.debug ('Host IP: ',hostIP)
cluster = Framework.getDestinationAttribute('cluster')
hostOS = Framework.getDestinationAttribute('host_os')
hostOS = hostOS or 'NA'
protocolName = Framework.getDestinationAttribute('Protocol')
hostId = Framework.getDestinationAttribute('hostId')
## Get Parameter Section
cldisp_command = Framework.getParameter('cldisp_command') or 'cldisp'
cllsif_command = Framework.getParameter('cllsif_command') or 'cllsif'
try:
client = Framework.createClient()
shell = ShellUtils(client)
# If we get good client connection , run the client commands to get the Application information for the cluster
HostOSH = modeling.createOshByCmdbIdString('host', hostId)
ClusterOSH = getclusterOSH(cluster)
appDictionary = getapplicationInfo(shell, cldisp_command, Framework)
resourceDictionary = getresourceinfo(shell, cllsif_command)
OSHVResult.addAll(createserviceapplicationOSH (shell, appDictionary, resourceDictionary, HostOSH, ClusterOSH, Framework))
client.close()
except JavaException, ex:
strException = ex.getMessage()
logger.debugException('')
errormessages.resolveAndReport(strException, protocolName, Framework)
def parse_oam_policy(policy_content):
"""
Parse oam policy.xml to get oam policies
:param policy_content:
:return: list of oam policy
:rtype: Policy
"""
logger.debug('parse oam policy.xml')
oam_policies = []
root = _buildDocumentForXpath(policy_content, 0)
xpath = _getXpath()
policies = xpath.evaluate('//AuthenticationPolicy | //AuthorizationPolicy', root, XPathConstants.NODESET)
for i in range(0, policies.getLength()):
policy = policies.item(i)
policy_type = policy.getNodeName()
policy_name = xpath.evaluate('name', policy, XPathConstants.STRING)
applicationDomainName = xpath.evaluate('applicationDomainName', policy, XPathConstants.STRING)
successRedirectURL = xpath.evaluate('successRedirectURL', policy, XPathConstants.STRING)
failureRedirectURL = xpath.evaluate('failureRedirectURL', policy, XPathConstants.STRING)
oam_policy = Policy(
policy_type, policy_name, applicationDomainName, successRedirectURL, failureRedirectURL, [])
oam_policies.append(oam_policy)
resources = xpath.evaluate('Resources/Resource', policy, XPathConstants.NODESET)
for j in range(0, resources.getLength()):
resource = resources.item(j)
resource_name = xpath.evaluate('name', resource, XPathConstants.STRING)
protectionLevel = xpath.evaluate('protectionLevel', resource, XPathConstants.STRING)
resourceURL = xpath.evaluate('resourceURL', resource, XPathConstants.STRING)
oam_resource = Resource(resource_name, protectionLevel, resourceURL)
oam_policy.add_resource(oam_resource)
logger.debug('find policy: %s' % oam_policy)
return oam_policies
def DiscoveryMain(Framework):
connectionDataManager = None
try:
logger.debug('Replicating topology from HP OneView')
connectionDataManager = FrameworkBasedConnectionDataManager(Framework)
if not connectionDataManager.validate():
return
mappingFileFolder = os.path.join(CollectorsParameters.BASE_PROBE_MGR_DIR,
CollectorsParameters.getDiscoveryConfigFolder(), HP_ONE_VIEW_CONFIG_FOLDER)
mappingFileManager = OneviewMappingFileManager(mappingFileFolder)
mappingFile = getMappingFileFromFramework(Framework)
if mappingFile:
return replicateTopologyUsingMappingFile(os.path.join(mappingFileFolder, mappingFile),
connectionDataManager, mappingFileManager)
else:
Framework.reportError('No mapping file found.')
logger.errorException("No mapping file found.")
except:
Framework.reportError('Failed to pull data from OneView.')
logger.errorException('Failed to pull data from OneView.')
finally:
if connectionDataManager:
connectionDataManager.closeClient()
def clear_history_tables():
logger.debug(u"PlexPy Database :: Deleting all session_history records... No turning back now bub.")
monitor_db = MonitorDatabase()
monitor_db.action('DELETE FROM session_history')
monitor_db.action('DELETE FROM session_history_media_info')
monitor_db.action('DELETE FROM session_history_metadata')
monitor_db.action('VACUUM')
def start(self):
from scheduler import Tree
loog = self.addLog("stdio")
self.pending = 0
properties = self.build.getProperties()
self.rendered_tree = tree = properties.render(self.treename)
l10nbuilds = properties.render(self.l10nbuilds)
cp = ConfigParser()
cp.read(l10nbuilds)
repo = cp.get(tree, "repo")
branch = cp.get(tree, "mozilla")
path = cp.get(tree, "l10n.ini")
l10nbranch = cp.get(tree, "l10n")
locales = cp.get(tree, "locales")
if locales == "all":
alllocales = "yes"
else:
alllocales = "no"
properties.update({"locales": filter(None, locales.split())}, "Build")
self.tree = Tree(self.rendered_tree, repo, branch, l10nbranch, path)
loog.addStdout("Loading l10n.inis for %s\n" % self.rendered_tree)
logger.debug(
"scheduler.l10n.tree", "Loading l10n.inis for %s, alllocales: %s" % (self.rendered_tree, alllocales)
)
self.loadIni(repo, branch, path, alllocales)
def osh_createDb2Tablespace(db2SubsystemOsh, tb):
str_name = 'name'
if UCMDB_VERSION < 9:
str_name = 'data_name'
if isNotNull(tb) and isNotNull(tb[0]):
tbOsh = ObjectStateHolder('mainframe_db2_tablespace')
tbOsh.setAttribute(str_name, tb[0])
tbOsh.setAttribute('dbtablespace_status', tb[1])
tbOsh.setAttribute('type', tb[2])
tbOsh.setAttribute('encoding_scheme', tb[3])
tbOsh.setAttribute('dbtablespace_initialextent', tb[4])
if isNotNull(tb[5]) and isnumeric(tb[5]):
tbOsh.setIntegerAttribute('max_dataset_size', int(tb[5]))
if isNotNull(tb[6]) and isnumeric(tb[6]):
tbOsh.setIntegerAttribute('number_tables', int(tb[6]))
if isNotNull(tb[7]) and isnumeric(tb[7]):
tbOsh.setIntegerAttribute('number_partitions', int(tb[7]))
try:
if len(tb[8]) > 19:
tb[8] = tb[8][0:18]
created = modeling.getDateFromString(tb[8], 'yyyy-MM-dd-kk.mm.ss', None)
tbOsh.setDateAttribute('create_date', created)
except:
logger.debug("Ignoring create_date. Unable to parse date string")
tbOsh.setContainer(db2SubsystemOsh)
return tbOsh
return None
def __init__(self, filename=":memory:", flags="r", mode=None, tablename="shelf"):
# XXX add flag/mode handling
# c -- create if it doesn't exist
# n -- new empty
# w -- open existing
# r -- readonly
logger.debug("Opening %s with flags %s" % (filename, flags))
self.conn = ClosedConnection()
dropIfExists = False
readOnly = False
if flags is not None:
if "n" in flags:
dropIfExists = True
if "r" in flags:
readOnly = True
if readOnly:
if not os.path.isfile(filename):
raise NoSuchFileError("%s doesn't exist" % (filename,))
self.tablename = tablename
self.filename = filename
self.conn = sqlite3.connect(filename)
self.conn.text_factory = str
if not readOnly:
if dropIfExists:
self.conn.execute("drop table if exists %s" % (self.tablename,))
MAKE_SHELF = "CREATE TABLE IF NOT EXISTS %s (key TEXT PRIMARY KEY, value TEXT NOT NULL)" % self.tablename
self.conn.execute(MAKE_SHELF)
self.conn.commit()
def create_necessary_paths(filename):
try:
(path,name) = os.path.split(filename)
logger.debug("Creating dir: "+path)
os.makedirs( path)
except:
pass
def envisalink_proxy(self, eventType, type, parameters, *args):
try:
res = yield self._connection.write(parameters)
logger.debug('PROXY > '+parameters.strip())
except StreamClosedError:
#we don't need to handle this, the callback has been set for closed connections.
pass
def queryRegistry(self, client, regKey, valueName):
if not (client and regKey and valueName):
logger.warn('registry query is incomplete')
return
logger.debug('RegistryBasedPlugin.queryRegistry')
ntcmdErrStr = 'Remote command returned 1(0x1)'
queryStr = ' query "%s" /v "%s"' % (regKey, valueName)
system32Link = client.createSystem32Link() or ''
buffer = client.execCmd(system32Link + "reg.exe" + queryStr)
if client.getLastCmdReturnCode() != 0 or buffer.find(ntcmdErrStr) != -1:
localFile = CollectorsParameters.BASE_PROBE_MGR_DIR + CollectorsParameters.getDiscoveryResourceFolder() + CollectorsParameters.FILE_SEPARATOR + 'reg_mam.exe'
remoteFile = client.copyFileIfNeeded(localFile)
if not remoteFile:
logger.warn('Failed copying reg_mam.exe to the destination')
return
buffer = client.execCmd(remoteFile + queryStr)
if not buffer or client.getLastCmdReturnCode() != 0:
logger.warn("Failed getting registry info.")
return
match = re.search(r'%s\s+%s\s+\w+\s+(.*)' % (regKey.replace('\\', '\\\\'), valueName), buffer, re.I)
client.removeSystem32Link()
if match:
val = match.group(1)
return val.strip()
logger.warn('Cannot parse registry key')
def getProps(Framework):
platform = Framework.getProperty(InventoryUtils.STATE_PROPERTY_PLATFORM)
architecture = Framework.getProperty(InventoryUtils.STATE_PROPERTY_ARCHITECTURE)
logger.debug('Platform: [', platform, '], architecture [', architecture, ']')
# We don't care for previous datadir and tempdir on windows
if str(platform) == "windows":
Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
else:
try:
client = Framework.getConnectedClient()
clientOptions = client.getOptionsMap()
envVariables = client.getEnvironmentVariables()
dataFolder = clientOptions.get(AgentUtils.DATA_DIR_OPTION)
tempFolder = envVariables.get(AgentUtils.TEMP_DIR_OPTION)
Framework.setProperty(AgentUtils.DATA_DIR_OPTION, dataFolder)
Framework.setProperty(AgentUtils.TEMP_DIR_OPTION, tempFolder)
logger.debug('Datadir option received from DDMI is [', dataFolder,
'] and the tempdir is [', tempFolder, ']')
Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
except:
Framework.setStepExecutionStatus(WorkflowStepStatus.FAILURE)
def Display(self, node, _detached):
if not node or node != self.lastNode:
self.storeLastItem()
if not self.prepare(node):
return
node=self.lastNode
self.control.AddColumn(xlt("Name"), 10)
self.control.AddColumn(xlt("Type"), 5)
self.control.AddColumn(xlt("Values"), 40)
self.control.AddColumn(xlt("TTL"), 5)
self.RestoreListcols()
for other in sorted(node.others.keys()):
rdss=node.others[other]
for rds in rdss.values():
icon=node.GetImageId('other')
dnstype=rdatatype.to_text(rds.rdtype)
name=other
for rd in rds:
values=[]
for slot in rd.__slots__:
value=eval("rd.%s" % slot)
if isinstance(value, list):
if len(value) > 1:
logger.debug("Value list dimensions > 1: %s", str(value))
value=" ".join(value)
values.append("%s=%s" % (slot, value))
self.control.AppendItem(icon, [name, dnstype, ", ".join(values), floatToTime(rds.ttl, -1)])
icon=0
name=""
dnstype=""
self.restoreLastItem()
def discoverDomainAdministrativeIps(allservers, dnsResolver):
r''' Discover administrative IP address in two ways
# 1) find server with admin-role and get server's IP or hostname (role contains port)
# 2) try to find the same information between managed-servers
@types: list[jee.Server], jee.DnsResolver -> list[str]'''
domainIpAddresses = []
isAdminServer = lambda server: server.hasRole(jee.AdminServerRole)
isManagedServer = lambda server: server.hasRole(weblogic.ManagedServerRole)
logger.debug( 'all servers', allservers )
adminServers = filter(isAdminServer, allservers)
logger.debug( 'admin servers', adminServers )
if adminServers:
adminServerAddresses = (adminServers[0].hostname, adminServers[0].ip.value())
domainIpAddresses = getIpsResolveIfNeeded(adminServerAddresses, dnsResolver)
if not adminServers[0].ip.value():
# in case there is only one resolved ip address we can set it in server
# otherwise problem to chose such
adminServers[0].ip.set(domainIpAddresses[0])
# case when failed to get admin-server IPs or admin-servers are not found
# 2)
if not domainIpAddresses:
# lets gather all IP addresses declared in managed servers
adminServerAddresses = {}
for server in filter(isManagedServer, allservers):
managedServerRole = server.getRole(weblogic.ManagedServerRole)
adminServerAddresses[managedServerRole.endpoint.getAddress()] = 1
# find IPs among addresses
domainIpAddresses = getIpsResolveIfNeeded(adminServerAddresses.keys(), dnsResolver)
return domainIpAddresses
def do_connect(self, reconnect = False):
# Create the socket and connect to the server
if reconnect == True:
logger.warning('Connection failed, retrying in '+str(self._retrydelay)+ ' seconds')
yield gen.sleep(self._retrydelay)
while self._connection == None:
logger.debug('Connecting to {}:{}'.format(config.ENVISALINKHOST, config.ENVISALINKPORT))
try:
self._connection = yield self.tcpclient.connect(config.ENVISALINKHOST, config.ENVISALINKPORT)
self._connection.set_close_callback(self.handle_close)
except StreamClosedError:
#failed to connect, but got no connection object so we will loop here
logger.warning('Connection failed, retrying in '+str(self._retrydelay)+ ' seconds')
yield gen.sleep(self._retrydelay)
continue
try:
line = yield self._connection.read_until(self._terminator)
except StreamClosedError:
#in this state, since the connection object isnt none, its going to throw the callback for handle_close so we just bomb out.
#and let handle_close deal with this
return
logger.debug("Connected to %s:%i" % (config.ENVISALINKHOST, config.ENVISALINKPORT))
self.handle_line(line)
def init(mode = ""):
logger.debug("Initializing...")
app = main.application()
registry.setValue("application", app)
error = None
try:
isfunc = eval("type(app." + mode + "Action)")
except:
isfunc = ""
pass
if (mode != "" and str(isfunc) == "<type 'instancemethod'>"):
registry.setValue("mode", mode)
error = eval("app." + mode + "Action()")
else:
registry.setValue("mode", "main")
error = app.mainAction()
if (error is not None):
logger.logException(error)
# Destroy window event
try:
eval("app.destroy()")
except:
pass
return False
def _makedirs(self, directories, basedir):
logger.debug("basedir: "+basedir)
for dir in directories:
curdir = os.path.join(basedir, dir)
if not os.path.exists(curdir):
logger.debug("Creating dir: "+curdir)
os.mkdir(curdir)
def process(self):
if self.is_valid_session():
ap = activity_processor.ActivityProcessor()
db_session = ap.get_session_by_key(
session_key=self.get_session_key())
this_state = self.timeline['state']
this_key = str(self.timeline['ratingKey'])
# If we already have this session in the temp table, check for state changes
if db_session:
# Re-schedule the callback to reset the 5 minutes timer
schedule_callback('session_key-{}'.format(
self.get_session_key()),
func=force_stop_stream,
args=[self.get_session_key()],
minutes=5)
last_state = db_session['state']
last_key = str(db_session['rating_key'])
# Make sure the same item is being played
if this_key == last_key:
# Update the session state and viewOffset
if this_state == 'playing':
# Update the session in our temp session table
# if the last set temporary stopped time exceeds 15 seconds
if int(time.time()) - db_session['stopped'] > 60:
self.update_db_session()
# Start our state checks
if this_state != last_state:
if this_state == 'paused':
self.on_pause()
elif last_state == 'paused' and this_state == 'playing':
self.on_resume()
elif this_state == 'stopped':
self.on_stop()
elif this_state == 'buffering':
self.on_buffer()
elif this_state == 'paused':
# Update the session last_paused timestamp
self.on_pause(still_paused=True)
# If a client doesn't register stop events (I'm looking at you PHT!) check if the ratingKey has changed
else:
# Manually stop and start
# Set force_stop so that we don't overwrite our last viewOffset
self.on_stop(force_stop=True)
self.on_start()
# Monitor if the stream has reached the watch percentage for notifications
# The only purpose of this is for notifications
if this_state != 'buffering':
progress_percent = helpers.get_percent(
self.timeline['viewOffset'], db_session['duration'])
watched_percent = {
'movie': plexpy.CONFIG.MOVIE_WATCHED_PERCENT,
'episode': plexpy.CONFIG.TV_WATCHED_PERCENT,
'track': plexpy.CONFIG.MUSIC_WATCHED_PERCENT,
'clip': plexpy.CONFIG.TV_WATCHED_PERCENT
}
if progress_percent >= watched_percent.get(
db_session['media_type'], 101):
watched_notifiers = notification_handler.get_notify_state_enabled(
session=db_session,
notify_action='on_watched',
notified=False)
if watched_notifiers:
logger.debug(
u"Tautulli ActivityHandler :: Session %s watched."
% str(self.get_session_key()))
for d in watched_notifiers:
plexpy.NOTIFY_QUEUE.put({
'stream_data':
db_session.copy(),
'notifier_id':
d['notifier_id'],
'notify_action':
'on_watched'
})
else:
# We don't have this session in our table yet, start a new one.
if this_state != 'buffering':
self.on_start()
# Schedule a callback to force stop a stale stream 5 minutes later
schedule_callback('session_key-{}'.format(
self.get_session_key()),
func=force_stop_stream,
args=[self.get_session_key()],
minutes=5)
def weakly_supervision_train():
device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
print(device)
model = maskrcnn_resnet50_fpn(pretrained=True)
in_features = model.roi_heads.box_predictor.cls_score.in_features
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, 7)
in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels
hidden_layer = 256
model.roi_heads.mask_predictor = MaskRCNNPredictor(in_features_mask,
hidden_layer, 7)
model.load_state_dict(
torch.load("../model/mask_rcnn_5_2_002.pth")) #加载已训练的模型
model.to(device)
model.open_weakly_supervision_train() #打开弱监督训练方式
for name, params in model.named_parameters():
if 'mask' not in name: #冻结mask分支以外的参数
params.requires_grad = False
optimizer = torch.optim.SGD(model.parameters(),
lr=0.005,
momentum=0.9,
weight_decay=0.0005)
num_epochs = 5
batch_size = 2
loss_sum = 0
data = SeashipDataset("../../SeaShips", None)
with open("./train_data", "r") as f:
lines = f.readlines()
train_list = [int(line) for line in lines]
print(len(train_list))
for epoch in range(num_epochs):
for idx in range(0, len(train_list), batch_size):
try:
imgs = []
targets = []
for i in range(idx, idx + batch_size):
img, target = data.getitem2(train_list[i] - 1, epoch)
imgs.append(F.to_tensor(img).to(device))
target = {k: v.to(device) for k, v in target.items()}
targets.append(target)
original_image_sizes = [img.shape[-2:] for img in imgs]
loss_dict, result = model.forward(imgs, targets)
losses = sum(loss for loss in loss_dict.values())
loss_sum += losses
#print(result)
optimizer.zero_grad()
losses.backward()
optimizer.step()
for j, res in enumerate(result):
scores = res['scores'].cpu().detach().numpy()
masks = res["masks"].cpu()
#print(masks[0].shape)
index = np.where(scores > 0.9) #只要9分以上的
masks = masks[index]
masks = torch.where(masks > 0.5, torch.full_like(masks, 1),
torch.full_like(masks, 0))
m = torch.zeros(original_image_sizes[0])
for mask in masks:
m += mask[0]
m = torch.where(m > 0.5, torch.full_like(m, 1),
torch.full_like(m, 0))
img_mask = TensorToPIL(m)
data.updatemask(idx + j, img_mask, epoch)
except:
logger.error(str(traceback.format_exc()))
if idx % 10 == 0:
#print("[%d]rpn_loss: %f" %(idx, loss_sum))
logger.debug("[%d]total_loss: %f" % (idx, loss_sum))
loss_sum = 0
torch.save(model.state_dict(), "./mask_rcnn_weakly_5_2_002.pth")
def train_mask():
os.environ["CUDA_VISIBLE_DEVICES"] = "2"
device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
print(device)
model = maskrcnn_resnet50_fpn(pretrained=True)
in_features = model.roi_heads.box_predictor.cls_score.in_features
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, 7)
in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels
hidden_layer = 256
model.roi_heads.mask_predictor = MaskRCNNPredictor(in_features_mask,
hidden_layer, 7)
model.to(device)
model.train()
data = SeashipDataset("../../SeaShips", None)
optimizer = torch.optim.SGD(model.parameters(),
lr=0.002,
momentum=0.9,
weight_decay=0.0005)
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=3,
gamma=0.1)
num_epochs = 5
batch_size = 2
loss_sum = 0
loss_classifier = 0
loss_box_reg = 0
loss_mask = 0
with open("./train_data", "r") as f:
lines = f.readlines()
train_list = [int(line) for line in lines]
print(len(train_list))
for epoch in range(num_epochs):
lr_scheduler = None
if epoch == 0:
warmup_factor = 1. / 1000
warmup_iters = min(1000, 5000 - 1)
lr_scheduler = utils.warmup_lr_scheduler(optimizer, warmup_iters,
warmup_factor)
for idx in range(0, len(train_list), batch_size):
try:
imgs = []
targets = []
for i in range(idx, idx + batch_size):
img, target = data.__getitem__(train_list[i] - 1)
imgs.append(F.to_tensor(img).to(device))
target = {k: v.to(device) for k, v in target.items()}
targets.append(target)
loss_dict = model.forward(imgs, targets)
losses = sum(loss for loss in loss_dict.values())
loss_sum += losses
#loss_classifier += loss_dict['loss_classifier'].values()
#loss_box_reg += loss_dict['loss_box_reg'].values()
#loss_mask += loss_dict['loss_mask'].values()
optimizer.zero_grad()
losses.backward()
optimizer.step()
if lr_scheduler is not None:
lr_scheduler.step()
except:
logger.error(str(traceback.format_exc()))
if idx % 12 == 0:
logger.debug("[%d]total_loss: %f" % (idx, loss_sum))
#logger.debug("[%d]loss: %f loss_classifier: %f loss_box_reg: %f loss_mask: %f" %(idx, loss_sum, loss_classifier, loss_box_reg, loss_mask))
loss_sum = 0
#loss_classifier = 0
#loss_box_reg = 0
#loss_mask = 0
torch.save(model.state_dict(), "../model/mask_rcnn_5_2_002.pth")
logger.debug("train successfully!")\
'100000')
client = Framework.createClient(props)
if client is None:
raise Exception, 'Failed to create NTCMD client'
except Exception, ex:
strException = ex.getMessage()
errormessages.resolveAndReport(strException, protocolName, Framework)
except:
strException = logger.prepareJythonStackTrace('')
errormessages.resolveAndReport(strException, protocolName, Framework)
else:
try:
clientShUtils = shellutils.ShellUtils(client)
OSHVResult.addAll(
NTCMD_HR_REG_Software_Lib.doSoftware(clientShUtils, hostOSH))
except Exception, ex:
strException = ex.getMessage()
errormessages.resolveAndReport(strException, protocolName,
Framework)
except:
strException = logger.prepareJythonStackTrace('')
errormessages.resolveAndReport(strException, protocolName,
Framework)
try:
clientShUtils and clientShUtils.clientClose()
except:
logger.debugException('')
logger.debug('Failed disconnecting from shell agent')
return OSHVResult
def parseTNSNames(tns_buffer, db_domain, shell=None):
tns_entries = []
tns_buffer = tns_buffer.upper()
tns_buffer = stripCommentLines(tns_buffer, '#')
logger.debug('tns_buffer')
logger.debug(tns_buffer)
tns_entries_str = findTNSEntriesStr(tns_buffer)
if tns_entries_str == []:
return [] # error, no entries
startPattern = Pattern('([\w\d.]*)\s*=\s*\(DESCRIPTION')
for tns_entry_str in tns_entries_str:
host_names = getTNSAttributeList(tns_entry_str, ['HOST'])
for host_name in host_names:
tns_entry = []
logger.debug('tns_entry_str', tns_entry_str)
match = startPattern.matcher(tns_entry_str)
if match.find() == 1:
tns_name = string.strip(match.group(1))
logger.debug('tns_name', tns_name)
tns_entry += [tns_name]
logger.debug('host_name', host_name)
tns_entry += [host_name]
port = getTNSAttribute(tns_entry_str, ['PORT'])
logger.debug('port', port)
tns_entry += [port]
sid = getTNSAttribute(tns_entry_str, ['SID'])
if sid == '':
sid = getTNSAttribute(tns_entry_str,
['SERVICE_NAME', 'service_name'])
if sid == '':
sid = getTNSAttribute(tns_entry_str, ['GLOBAL_DBNAME'])
tns_name = stripDomain(tns_name, db_domain)
sid = stripDomain(sid, db_domain)
logger.debug('sid', sid)
tns_entry += [sid]
tns_entry += [tns_name]
host_ip = ''
if shell:
try:
resolver = netutils.DNSResolver(shell)
ips = resolver.resolveIpByNsLookup(host_name)
host_ip = ips and ips[
0] or resolver.resolveHostIpByHostsFile(host_name)
except:
logger.warn('Failed to resolve host ip throught nslookup')
host_ip = host_name
else:
host_ip = netutils.getHostAddress(host_name, host_name)
tns_entry += [host_ip]
tns_entries += [tns_entry]
logger.debug(tns_entry)
return tns_entries
#!/usr/bin/python
import logger
import logging
import requests
logger.debug('123')
logger.error('Failed...')
logger.cprint(logger.LIGHT_CYAN, 'cprint!')
logging.debug('debug 123')
#logging.info('info 123')
#logging.warning('warning 123')
import urllib2
url = 'http://10.88.15.168:9200/File/FileDownloadService.aspx?locationID=7&fileID=d08ede40-4fe5-4ebf-b548-b3225ea815bf'
file_name = urllib2.unquote(url).decode('utf8').split('/')[-1]
print(file_name)
r = requests.get(url, stream=True)
#print r.headers['Location']
print r.headers["Content-Length"]
def process(self):
if self.is_item():
global RECENTLY_ADDED_QUEUE
rating_key = self.get_rating_key()
media_types = {
1: 'movie',
2: 'show',
3: 'season',
4: 'episode',
8: 'artist',
9: 'album',
10: 'track'
}
identifier = self.timeline.get('identifier')
state_type = self.timeline.get('state')
media_type = media_types.get(self.timeline.get('type'))
section_id = self.timeline.get('sectionID', 0)
title = self.timeline.get('title', 'Unknown')
metadata_state = self.timeline.get('metadataState')
media_state = self.timeline.get('mediaState')
queue_size = self.timeline.get('queueSize')
# Return if it is not a library event (i.e. DVR EPG event)
if identifier != 'com.plexapp.plugins.library':
return
# Add a new media item to the recently added queue
if media_type and section_id > 0 and \
((state_type == 0 and metadata_state == 'created')): # or \
#(plexpy.CONFIG.NOTIFY_RECENTLY_ADDED_UPGRADE and state_type in (1, 5) and \
#media_state == 'analyzing' and queue_size is None)):
if media_type in ('episode', 'track'):
metadata = self.get_metadata()
if metadata:
grandparent_rating_key = int(
metadata['grandparent_rating_key'])
parent_rating_key = int(metadata['parent_rating_key'])
grandparent_set = RECENTLY_ADDED_QUEUE.get(
grandparent_rating_key, set())
grandparent_set.add(parent_rating_key)
RECENTLY_ADDED_QUEUE[
grandparent_rating_key] = grandparent_set
parent_set = RECENTLY_ADDED_QUEUE.get(
parent_rating_key, set())
parent_set.add(rating_key)
RECENTLY_ADDED_QUEUE[parent_rating_key] = parent_set
RECENTLY_ADDED_QUEUE[rating_key] = set(
[grandparent_rating_key])
logger.debug(
u"Tautulli TimelineHandler :: Library item '%s' (%s, grandparent %s) added to recently added queue."
% (title, str(rating_key),
str(grandparent_rating_key)))
# Schedule a callback to clear the recently added queue
schedule_callback(
'rating_key-{}'.format(grandparent_rating_key),
func=clear_recently_added_queue,
args=[grandparent_rating_key],
seconds=plexpy.CONFIG.NOTIFY_RECENTLY_ADDED_DELAY)
elif media_type in ('season', 'album'):
metadata = self.get_metadata()
if metadata:
parent_rating_key = int(metadata['parent_rating_key'])
parent_set = RECENTLY_ADDED_QUEUE.get(
parent_rating_key, set())
parent_set.add(rating_key)
RECENTLY_ADDED_QUEUE[parent_rating_key] = parent_set
logger.debug(
u"Tautulli TimelineHandler :: Library item '%s' (%s , parent %s) added to recently added queue."
% (title, str(rating_key), str(parent_rating_key)))
# Schedule a callback to clear the recently added queue
schedule_callback(
'rating_key-{}'.format(parent_rating_key),
func=clear_recently_added_queue,
args=[parent_rating_key],
seconds=plexpy.CONFIG.NOTIFY_RECENTLY_ADDED_DELAY)
else:
queue_set = RECENTLY_ADDED_QUEUE.get(rating_key, set())
RECENTLY_ADDED_QUEUE[rating_key] = queue_set
logger.debug(
u"Tautulli TimelineHandler :: Library item '%s' (%s) added to recently added queue."
% (title, str(rating_key)))
# Schedule a callback to clear the recently added queue
schedule_callback(
'rating_key-{}'.format(rating_key),
func=clear_recently_added_queue,
args=[rating_key],
seconds=plexpy.CONFIG.NOTIFY_RECENTLY_ADDED_DELAY)
# A movie, show, or artist is done processing
elif media_type in ('movie', 'show', 'artist') and section_id > 0 and \
state_type == 5 and metadata_state is None and queue_size is None and \
rating_key in RECENTLY_ADDED_QUEUE:
logger.debug(
u"Tautulli TimelineHandler :: Library item '%s' (%s) done processing metadata."
% (title, str(rating_key)))
# An item was deleted, make sure it is removed from the queue
elif state_type == 9 and metadata_state == 'deleted':
if rating_key in RECENTLY_ADDED_QUEUE and not RECENTLY_ADDED_QUEUE[
rating_key]:
logger.debug(
u"Tautulli TimelineHandler :: Library item %s removed from recently added queue."
% str(rating_key))
del_keys(rating_key)
# Remove the callback if the item is removed
schedule_callback('rating_key-{}'.format(rating_key),
remove_job=True)
def __readPluginDescriptors(self):
allDescriptors = PluginsPackageConfigFile.getAllPluginDescriptors()
logger.debug("PluginEngine found %d plugins" % len(allDescriptors))
return allDescriptors
def DiscoveryMain(Framework):
resultVector = ObjectStateHolderVector()
ipAddress = Framework.getDestinationAttribute(
DestinationProperty.IP_ADDRESS)
if not ipAddress:
msg = errormessages.makeErrorMessage(na.Protocol.DISPLAY,
message="Invalid IP address")
errorObject = errorobject.createError(
errorcodes.INTERNAL_ERROR_WITH_PROTOCOL_DETAILS,
[na.Protocol.DISPLAY, msg], msg)
logger.reportErrorObject(errorObject)
return resultVector
credentialsId = Framework.getParameter(DestinationProperty.CREDENTIALS_ID)
if not credentialsId:
msg = errormessages.makeErrorMessage(
na.Protocol.DISPLAY, pattern=errormessages.ERROR_NO_CREDENTIALS)
errorObject = errorobject.createError(
errorcodes.NO_CREDENTIALS_FOR_TRIGGERED_IP, [na.Protocol.DISPLAY],
msg)
logger.reportErrorObject(errorObject)
return resultVector
queryTopologyPerDevice = _getBooleanParameter(
JobParameter.QUERY_TOPOLOGY_PER_DEVICE, Framework, False)
discovererClass = na_discover.SingleRequestsNaDiscoverer
if queryTopologyPerDevice:
discovererClass = na_discover.NaDiscovererPerDevice
reportDeviceConfigs = _getBooleanParameter(
JobParameter.REPORT_DEVICE_CONFIGS, Framework, False)
logger.debug('reportDeviceConfigs: ', reportDeviceConfigs)
reportDeviceModules = _getBooleanParameter(
JobParameter.REPORT_DEVICE_MODULES, Framework, False)
logger.debug('reportDeviceModules:', reportDeviceModules)
client = None
try:
try:
client = na_discover.createJavaClient(Framework, ipAddress,
credentialsId)
logger.debug("Topology is discovered by '%s'" %
discovererClass.__name__)
discoverer = discovererClass(client, ipAddress, Framework)
discoverer.setDevicePageSize(500)
discoverer.setReportDeviceConfigs(reportDeviceConfigs)
discoverer.setReportDeviceModules(reportDeviceModules)
discoverResult = discoverer.discover()
if discoverResult:
devicesById, connectivitiesByDeviceId = discoverResult
reporter = na.NaReporter(Framework)
reporter.setBulkThreshold(10000)
reporter.setReportDeviceConfigs(reportDeviceConfigs)
reporter.setReportDeviceModules(reportDeviceModules)
reporter.report(devicesById, connectivitiesByDeviceId)
finally:
client and client.close()
except MissingSdkJarException, ex:
msg = errormessages.makeErrorMessage(
na.Protocol.DISPLAY,
message="Not all jar dependencies are found in class path")
errorObject = errorobject.createError(errorcodes.MISSING_JARS_ERROR,
[na.Protocol.DISPLAY, msg], msg)
logger.reportErrorObject(errorObject)
def discover(self,
domainName,
traHome,
tmpDir,
cleanTemp=1,
discoverJmsTopology=Boolean.TRUE):
r'@types: str, str, str, netutils.BaseDnsResolver, bool -> '
ts = Date().getTime()
ucmdbTmpDir = "/forUcmdb-%s" % ts
applications = []
# first check if tmpDir exists
fs = self.getFileSystem()
if fs.exists(tmpDir):
logger.debug(
'[' + __name__ +
':BusinessWorksApplicationDiscoverer.discover] %s directory exists'
% tmpDir)
tmpDir = fs.getFile(tmpDir)
logger.debug(
'[' + __name__ +
':BusinessWorksApplicationDiscoverer.discover] Permissions - %s, Path = %s'
% (tmpDir.permissions(), tmpDir.path))
# TODO check for permissions?
else:
logger.errorException(
'[' + __name__ +
':BusinessWorksApplicationDiscoverer.discover] %s directory does not exist'
% tmpDir)
# TODO: raise exception or log
return None
# get the tra version
appManageFileDir = "%sbin/" % traHome
if fs.exists(appManageFileDir):
appManageFile = fs.getFile("%sAppManage" % appManageFileDir)
logger.debug(
'[' + __name__ +
':BusinessWorksApplicationDiscoverer.discover] Found AppManage file path - %s'
% appManageFile.path)
# run the AppManage command for given domainName
tempDomainDir = "%s%s/%s" % (tmpDir.path, ucmdbTmpDir, domainName)
shell = self.getShell()
shell.execCmd("mkdir -p %s" % tempDomainDir)
if shell.getLastCmdReturnCode() == 0:
logger.debug(
'[' + __name__ +
':BusinessWorksApplicationDiscoverer.discover] Successfully created temp UCMDB directory: %s'
% tempDomainDir)
shell.execCmd("cd %s" % appManageFileDir)
if shell.getLastCmdReturnCode() == 0:
buffer = shell.execCmd("pwd")
logger.debug(
'[' + __name__ +
':BusinessWorksApplicationDiscoverer.discover] Changed working directory to = %s'
% buffer)
if self.__executeAppManage(tempDomainDir, domainName):
# let's get the applications now
appMap = self.__getAppConfigsMap(shell, tempDomainDir)
for (appName, xmlFile) in appMap.items():
#logger.debug("%s --> %s" % (fileName, xmlFile))
xmlFilePath = "%s/%s" % (tempDomainDir, xmlFile)
if fs.exists(xmlFilePath):
folder = self.getPathUtils().dirName(
appName) or ""
name = self.getPathUtils().baseName(
appName) or ""
if name:
application = tibco.Application(
name, folder)
applications.append(application)
xmlFile = fs.getFile(
xmlFilePath, [FileAttrs.CONTENT])
if shell.getLastCmdReturnCode() == 0:
jmsDiscoverer = JmsDiscoverer()
try:
document = XmlParser().getDocument(
xmlFile.content,
namespaceAware=0)
except JException, je:
logger.warnException(
"Failed to parse XML document. %s"
% str(je))
else:
if discoverJmsTopology == Boolean.TRUE:
tibco.each(
application.addJmsServer,
jmsDiscoverer.
discoverJmsServer(
document))
tibco.each(
application.addJmsQueue,
jmsDiscoverer.
discoverJmsQueues(
document))
tibco.each(
application.addJmsTopic,
jmsDiscoverer.
discoverJmsTopics(
document))
tibco.each(
application.addAdapter,
self.__adapterDiscoverer.
discover(document, name))
else:
logger.error(
'Failed to get content of %s' %
xmlFilePath)
# remove temporary directory
logger.debug("Clean temp folder")
if cleanTemp:
try:
# fs.removeFiles(self.__createdFiles)
self.__removeTempUcmdbDirectory(
"%s%s" % (tmpDir.path, ucmdbTmpDir))
except Exception, ex:
logger.warnException(str(ex))
logger.reportWarning(
"Unable to delete temporary folder")
def extractReplacingMainFolder(self, file, dir, replacement):
logger.debug("file=%s" % file)
logger.debug("dir=%s" % dir)
if not dir.endswith(':') and not os.path.exists(dir):
os.mkdir(dir)
zf = zipfile.ZipFile(file)
self._createstructure(file, dir, replacement)
num_files = len(zf.namelist())
for name in zf.namelist():
logger.debug("name=%s" % name)
if not name.endswith('/'):
logger.debug("continue with file: " + name)
try:
(path, filename) = os.path.split(os.path.join(dir, name))
logger.debug("path=%s" % path)
logger.debug("name=%s" % name)
if replacement != '':
logger.debug("replace original folder: " + path)
path = path.replace(name[:name.find("/")], replacement)
logger.debug("replaced [new] folder: " + path)
os.makedirs(path)
except:
pass
newName = ''
if replacement != '':
logger.debug("replace original name: " + name)
newName = name.replace(name[:name.find("/")], replacement)
logger.debug("replaced [new] name: " + name)
outfilename = os.path.join(dir, newName)
logger.debug("outfilename=%s" % outfilename)
try:
outfile = open(outfilename, 'wb')
outfile.write(zf.read(name))
except:
logger.error("Something happened in file: " + outfilename)
def __logDecision(self, descriptorId, decision):
params = (descriptorId, self, decision)
logger.debug(" '%s': %r -> %s" % params)
def gyro_integrate(conn):
'''test gyros by integrating while rotating to the given rotations'''
conn.ref.send('set streamrate -1\n')
conn.test.send('set streamrate -1\n')
util.param_set(conn.ref, 'SR0_RAW_SENS', 20)
util.param_set(conn.test, 'SR0_RAW_SENS', 20)
logger.info("Starting gyro integration")
wait_quiescent(conn.refmav)
conn.discard_messages()
if ETE == 0:
util.set_servo(conn.refmav, YAW_CHANNEL, ROTATIONS['level'].chan1+200)
util.set_servo(conn.refmav, PITCH_CHANNEL, ROTATIONS['level'].chan2+200)
if ETE == 1:
ete = PixETE()
ete.position(45, 180)
time.sleep(1)
ete.rollspeed(4000)
ete.position(45, 0)
logger.info("Starting gyro motion")
start_time = time.time()
ref_tstart = None
test_tstart = [None]*3
ref_sum = Vector3()
test_sum = [Vector3(), Vector3(), Vector3()]
msgs = { 'RAW_IMU' : 0, 'SCALED_IMU2' : 1, 'SCALED_IMU3' : 2 }
while time.time() < start_time+20:
imu = conn.refmav.recv_match(type='RAW_IMU', blocking=False)
if imu is not None:
#gyro = util.gyro_vector(imu)
gyro = Vector3(degrees(imu.xgyro*-0.001), degrees(imu.ygyro*0.001), degrees(imu.zgyro*-0.001)) #phil change.... when running this check from back to top, I found that the reference IMU X and Z were reversed.... this hack makes this pass, but I suspect something else is wrong, this should be reverted when that is found.
tnow = imu.time_usec*1.0e-6
if ref_tstart is not None:
deltat = tnow - ref_tstart
ref_sum += gyro * deltat
ref_tstart = tnow
if time.time() - start_time > 2 and gyro.length() < GYRO_TOLERANCE:
break
imu = conn.testmav.recv_match(type=msgs.keys(), blocking=False)
if imu is not None:
idx = msgs[imu.get_type()]
gyro = util.gyro_vector(imu)
if imu.get_type().startswith("SCALED_IMU"):
tnow = imu.time_boot_ms*1.0e-3
else:
tnow = imu.time_usec*1.0e-6
if test_tstart[idx] is not None:
deltat = tnow - test_tstart[idx]
test_sum[idx] += gyro * deltat
test_tstart[idx] = tnow
logger.debug("Gyro ref sums: %s" % ref_sum)
logger.debug("Gyro test sum1: %s" % test_sum[0])
logger.debug("Gyro test sum2: %s" % test_sum[1])
logger.debug("Gyro test sum3: %s" % test_sum[2])
ete.yawspeed(5000)
ete.rollspeed(5000)
ete.position(0, 0)
wait_quiescent(conn.refmav)
for idx in range(3):
err = test_sum[idx] - ref_sum
if abs(err.x) > GYRO_SUM_TOLERANCE:
util.failure("X gyro %u error: %.1f" % (idx, err.x))
if abs(err.y) > GYRO_SUM_TOLERANCE:
util.failure("Y gyro %u error: %.1f" % (idx, err.y))
if abs(err.z) > GYRO_SUM_TOLERANCE:
util.failure("Z gyro %u error: %.1f" % (idx, err.z))
logger.debug("Gyro test finished")
def extract(self, file, dir):
logger.debug("file=%s" % file)
logger.debug("dir=%s" % dir)
if not dir.endswith(':') and not os.path.exists(dir):
os.mkdir(dir)
zf = zipfile.ZipFile(file)
self._createstructure(file, dir)
num_files = len(zf.namelist())
for name in zf.namelist():
logger.debug("name=%s" % name)
if not name.endswith('/'):
logger.debug("continue with file: " + name)
try:
(path, filename) = os.path.split(os.path.join(dir, name))
logger.debug("path=%s" % path)
logger.debug("name=%s" % name)
os.makedirs(path)
except:
pass
outfilename = os.path.join(dir, name)
logger.debug("outfilename=%s" % outfilename)
try:
outfile = open(outfilename, 'wb')
outfile.write(zf.read(name))
except:
logger.error("Something happened in file: " + name)
def cachePage(url,post=None,headers=[['User-Agent', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; es-ES; rv:1.9.2.12) Gecko/20101026 Firefox/3.6.12']],modoCache=CACHE_ACTIVA, timeout=socket.getdefaulttimeout()):
logger.info("[scrapertools.py] cachePage url="+url)
modoCache = config.get_setting("cache.mode")
'''
if config.get_platform()=="plex":
from PMS import HTTP
try:
logger.info("url="+url)
data = HTTP.Request(url)
logger.info("descargada")
except:
data = ""
logger.error("Error descargando "+url)
import sys
for line in sys.exc_info():
logger.error( "%s" % line )
return data
'''
# CACHE_NUNCA: Siempre va a la URL a descargar
# obligatorio para peticiones POST
if modoCache == CACHE_NUNCA or post is not None:
logger.info("[scrapertools.py] MODO_CACHE=2 (no cachear)")
try:
data = downloadpage(url,post,headers, timeout=timeout)
except:
data=""
# CACHE_SIEMPRE: Siempre descarga de cache, sin comprobar fechas, excepto cuando no está
elif modoCache == CACHE_SIEMPRE:
logger.info("[scrapertools.py] MODO_CACHE=1 (cachear todo)")
# Obtiene los handlers del fichero en la cache
cachedFile, newFile = getCacheFileNames(url)
# Si no hay ninguno, descarga
if cachedFile == "":
logger.debug("[scrapertools.py] No está en cache")
# Lo descarga
data = downloadpage(url,post,headers)
# Lo graba en cache
outfile = open(newFile,"w")
outfile.write(data)
outfile.flush()
outfile.close()
logger.info("[scrapertools.py] Grabado a " + newFile)
else:
logger.info("[scrapertools.py] Leyendo de cache " + cachedFile)
infile = open( cachedFile )
data = infile.read()
infile.close()
# CACHE_ACTIVA: Descarga de la cache si no ha cambiado
else:
logger.info("[scrapertools.py] MODO_CACHE=0 (automática)")
# Datos descargados
data = ""
# Obtiene los handlers del fichero en la cache
cachedFile, newFile = getCacheFileNames(url)
# Lo descarga
data = downloadpage(url,post,headers)
# Lo graba en cache
outfile = open(newFile,"w")
outfile.write(data)
outfile.flush()
outfile.close()
logger.info("[scrapertools.py] Grabado a " + newFile)
return data
if predict_rating > actual_rating:
result_sim.loc[common_index] -= learning_rate * user_ratings
else:
result_sim.loc[common_index] += learning_rate * user_ratings
################################################################################
# end of functions
################################################################################
logger.info('Start program: ' + __file__)
logger.info('Number of data sets: %d' % number_of_data_sets)
logger.info('Data sets dir: ' + data_sets_dir)
logger.info('Number of nearest neighbors: %d' % k)
logger.debug('======================================')
# loop throught data sets
for i in range(number_of_data_sets):
test_file = data_sets_dir + str(i) + '/test.csv'
train_file = data_sets_dir + str(i) + '/train.csv'
# load test and train data
logger.info('Test data file: ' + test_file)
logger.info('Train data file: ' + train_file)
logger.info('Start loading data...')
test_df = pd.read_csv(test_file)
train_df = pd.read_csv(train_file)
logger.info('Done loading')
test_productid_array = test_df.product_productid.unique()
def ApiCall(url, method, data, head):
requests.packages.urllib3.disable_warnings()
if url is None or not url.startswith("http"):
logger.error("invalid URL")
yn = logger.input("Change URL? [y/N] > ")
if yn == 'y' or yn == "Y":
url = logger.input("Enter endpoint URL > ")
updateProfileAttr("endpoint", url)
with open(LOCAL_FOLDER + ".endpoint", "w+") as f:
f.write(url)
else:
return
res = None
if head is not None:
if "content-type" in head or "Content-Type" in head:
headers = head
else:
head = json.dumps(head)
headers = '{"content-type": "application/json", ' + head[1:]
headers = json.loads(headers)
else:
headers = {'content-type': 'application/json'}
logger.debug("Sending requests to: " + url)
if method == "POST":
data = data
logger.debug("Request Payload: " + str(data))
try:
if getProfile()["proxy"] is not None:
proxies = { 'http': "http://" + getProfile()["proxy"], 'https': "http://" + getProfile()["proxy"] }
res = requests.post(url, data=data, headers=headers, proxies=proxies, verify=False)
else:
res = requests.post(url, data=data, headers=headers)
except urllib2.HTTPError as e:
logger.debug("RESPONSE: " + str(e))
return '{"status": "err"}'
except ValueError as e:
if str(e).startswith("Invalid header value"):
logger.error("Invalid session. Please, re-authenticate")
updateProfileAttr("isAuth", False)
updateProfileAttr("session", None)
logger.pressAnyKey()
return '{"status": "err"}'
else:
proxies = None
if getProfile()["proxy"] is not None:
# req.add_header('Host', '127.0.0.1')
proxies = {'http': "http://" + getProfile()["proxy"], 'https': "http://" + getProfile()["proxy"]}
try:
res = requests.get(url, headers=headers, proxies=proxies, verify=False)
except urllib2.HTTPError as e:
logger.debug ("RESPONSE: " + str(e))
return '{"status": "err"}'
except ValueError as e:
if str(e).startswith("Invalid header value"):
logger.error("Invalid session. Please, re-authenticate")
updateProfileAttr("isAuth", False)
updateProfileAttr("session", None)
logger.pressAnyKey()
return '{"status": "err"}'
if len(res.content) > 64000:
logger.debug("Response is too big")
else:
logger.debug("RESPONSE: " + res.content.encode('utf-8').strip())
return res.content
def optimise_attitude(conn, rotation, tolerance, timeout=25):
'''optimise attitude using servo changes'''
expected_roll = ROTATIONS[rotation].roll
expected_pitch = ROTATIONS[rotation].pitch
if ETE == 0:
chan1 = ROTATIONS[rotation].chan1
chan2 = ROTATIONS[rotation].chan2
elif ETE == 1:
chan1 = ROTATIONS_ETE[rotation].chan1
chan2 = ROTATIONS_ETE[rotation].chan2
attitude = wait_quiescent(conn.refmav)
if ETE == 1:
return True
time_start = time.time()
# we always do at least 2 tries. This means the attitude accuracy
# will tend to improve over time, while not adding excessive time
# per board
tries = 0
while time.time() < time_start+timeout:
#logger.info("============================= BEGIN ROTATIONS try=%s =================" % (tries))
dcm_estimated = Matrix3()
dcm_estimated.from_euler(attitude.roll, attitude.pitch, attitude.yaw)
droll = expected_roll
if droll is None:
droll = attitude.roll
else:
droll = radians(droll)
dpitch = radians(expected_pitch)
dcm_demanded = Matrix3()
dcm_demanded.from_euler(droll, dpitch, attitude.yaw)
(chan1_change, chan2_change) = gimbal_controller(dcm_estimated,
dcm_demanded, chan1)
(err_roll, err_pitch) = attitude_error(attitude, expected_roll, expected_pitch)
logger.info("optimise_attitude: %s err_roll=%.2f err_pitch=%.2f chan1=%u chan2=%u" % (rotation, err_roll, err_pitch, chan1, chan2))
if (tries > 0 and (abs(err_roll)+abs(err_pitch) < tolerance or
(abs(chan1_change)<1 and abs(chan2_change)<1))):
logger.debug("%s converged %.2f %.2f tolerance %.1f" % (rotation, err_roll, err_pitch, tolerance))
# update optimised rotations to save on convergence time for the next board
ROTATIONS[rotation].chan1 = chan1
ROTATIONS[rotation].chan2 = chan2
logger.debug("optimise_attitude: ROTATIONS[%s] chan1:%s chan2:%s" % (rotation, ROTATIONS[rotation].chan1, ROTATIONS[rotation].chan2) )
return True
chan1 += chan1_change
chan2 += chan2_change
if chan1 < 700 or chan1 > 2300 or chan2 < 700 or chan2 > 2300:
logger.debug("servos out of range - failed")
return False
util.set_servo(conn.refmav, YAW_CHANNEL, chan1)
util.set_servo(conn.refmav, PITCH_CHANNEL, chan2)
attitude = wait_quiescent(conn.refmav)
tries += 1
logger.error("timed out rotating to %s" % rotation)
return False
class state():
logger.debug('State Module Loaded')
@staticmethod
def init():
state.state = {}
events.register('alarm', state.update)
@staticmethod
def getDict():
return state.state
@staticmethod
def setVersion(version):
state.state['version'] = version
@staticmethod
def update(eventType, type, parameters, code, event, message,
defaultStatus):
if not type in state.state: state.state[type] = {'lastevents': []}
#keep the last state
try:
prev_status = state.state[type][parameters]['status']
except (IndexError, KeyError):
#if we are here, we've never seen this event type, parameter combination before
prev_status = None
# if this event has never generated 'state' before, populate the defaults
if not parameters in state.state[type]:
state.state[type][parameters] = {
'name':
config.ZONENAMES[parameters]
if type == 'zone' else config.PARTITIONNAMES[parameters],
'lastevents': [],
'status':
defaultStatus
}
#update the state
state.state[type][parameters]['status'] = dict(
state.state[type][parameters]['status'], **event['status'])
#if this is the first event in this zone/partition we've seen, don't do anything here.
if prev_status != None:
#if we've seen this before, check if it's changed
if prev_status == state.state[type][parameters]['status']:
logger.debug(
'Discarded event. State not changed. ({} {})'.format(
event['type'], parameters))
else:
events.put('statechange', type, parameters, code, event,
message, defaultStatus)
else:
events.put('stateinit', type, parameters, code, event, message,
defaultStatus)
#write event
state.state[type][parameters]['lastevents'].append({
'datetime':
str(datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")),
'code':
code,
'message':
message
})
#write to all events
state.state[type]['lastevents'].append({
'datetime':
str(datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")),
'code':
code,
'message':
message
})
def verify_directories_created():
import logger
import os
logger.info("streamondemand.core.config.verify_directories_created")
# Force download path if empty
download_path = get_setting("downloadpath")
if download_path=="":
download_path = os.path.join( get_data_path() , "downloads")
set_setting("downloadpath" , download_path)
# Force download list path if empty
download_list_path = get_setting("downloadlistpath")
if download_list_path=="":
download_list_path = os.path.join( get_data_path() , "downloads" , "list")
set_setting("downloadlistpath" , download_list_path)
# Force bookmark path if empty
bookmark_path = get_setting("bookmarkpath")
if bookmark_path=="":
bookmark_path = os.path.join( get_data_path() , "bookmarks")
set_setting("bookmarkpath" , bookmark_path)
# Create data_path if not exists
if not os.path.exists(get_data_path()):
logger.debug("Creating data_path "+get_data_path())
try:
os.mkdir(get_data_path())
except:
pass
# Create download_path if not exists
if not download_path.lower().startswith("smb") and not os.path.exists(download_path):
logger.debug("Creating download_path "+download_path)
try:
os.mkdir(download_path)
except:
pass
# Create download_list_path if not exists
if not download_list_path.lower().startswith("smb") and not os.path.exists(download_list_path):
logger.debug("Creating download_list_path "+download_list_path)
try:
os.mkdir(download_list_path)
except:
pass
# Create bookmark_path if not exists
if not bookmark_path.lower().startswith("smb") and not os.path.exists(bookmark_path):
logger.debug("Creating bookmark_path "+bookmark_path)
try:
os.mkdir(bookmark_path)
except:
pass
# Create library_path if not exists
if not get_library_path().lower().startswith("smb") and not os.path.exists(get_library_path()):
logger.debug("Creating library_path "+get_library_path())
try:
os.mkdir(get_library_path())
except:
pass
# Checks that a directory "xbmc" is not present on platformcode
old_xbmc_directory = os.path.join( get_runtime_path() , "platformcode" , "xbmc" )
if os.path.exists( old_xbmc_directory ):
logger.debug("Removing old platformcode.xbmc directory")
try:
import shutil
shutil.rmtree(old_xbmc_directory)
except:
pass
def check_github(scheduler=False, notify=False, use_cache=False):
plexpy.COMMITS_BEHIND = 0
if plexpy.CONFIG.GIT_TOKEN:
headers = {'Authorization': 'token {}'.format(plexpy.CONFIG.GIT_TOKEN)}
else:
headers = {}
version = github_cache('version', use_cache=use_cache)
if not version:
# Get the latest version available from github
logger.info('Retrieving latest version information from GitHub')
url = 'https://api.github.com/repos/%s/%s/commits/%s' % (
plexpy.CONFIG.GIT_USER, plexpy.CONFIG.GIT_REPO,
plexpy.CONFIG.GIT_BRANCH)
version = request.request_json(url,
headers=headers,
timeout=20,
validator=lambda x: type(x) == dict)
github_cache('version', github_data=version)
if version is None:
logger.warn(
'Could not get the latest version from GitHub. Are you running a local development version?'
)
return plexpy.CURRENT_VERSION
plexpy.LATEST_VERSION = version['sha']
logger.debug("Latest version is %s", plexpy.LATEST_VERSION)
# See how many commits behind we are
if not plexpy.CURRENT_VERSION:
logger.info(
'You are running an unknown version of Tautulli. Run the updater to identify your version'
)
return plexpy.LATEST_VERSION
if plexpy.LATEST_VERSION == plexpy.CURRENT_VERSION:
logger.info('Tautulli is up to date')
return plexpy.LATEST_VERSION
commits = github_cache('commits', use_cache=use_cache)
if not commits:
logger.info(
'Comparing currently installed version with latest GitHub version')
url = 'https://api.github.com/repos/%s/%s/compare/%s...%s' % (
plexpy.CONFIG.GIT_USER, plexpy.CONFIG.GIT_REPO,
plexpy.LATEST_VERSION, plexpy.CURRENT_VERSION)
commits = request.request_json(url,
headers=headers,
timeout=20,
whitelist_status_code=404,
validator=lambda x: type(x) == dict)
github_cache('commits', github_data=commits)
if commits is None:
logger.warn('Could not get commits behind from GitHub.')
return plexpy.LATEST_VERSION
try:
plexpy.COMMITS_BEHIND = int(commits['behind_by'])
logger.debug("In total, %d commits behind", plexpy.COMMITS_BEHIND)
except KeyError:
logger.info(
'Cannot compare versions. Are you running a local development version?'
)
plexpy.COMMITS_BEHIND = 0
if plexpy.COMMITS_BEHIND > 0:
logger.info('New version is available. You are %s commits behind' %
plexpy.COMMITS_BEHIND)
releases = github_cache('releases', use_cache=use_cache)
if not releases:
url = 'https://api.github.com/repos/%s/%s/releases' % (
plexpy.CONFIG.GIT_USER, plexpy.CONFIG.GIT_REPO)
releases = request.request_json(
url,
timeout=20,
whitelist_status_code=404,
validator=lambda x: type(x) == list)
github_cache('releases', github_data=releases)
if releases is None:
logger.warn('Could not get releases from GitHub.')
return plexpy.LATEST_VERSION
if plexpy.CONFIG.GIT_BRANCH == 'master':
release = next((r for r in releases if not r['prerelease']),
releases[0])
elif plexpy.CONFIG.GIT_BRANCH == 'beta':
release = next(
(r
for r in releases if not r['tag_name'].endswith('-nightly')),
releases[0])
elif plexpy.CONFIG.GIT_BRANCH == 'nightly':
release = next((r for r in releases), releases[0])
else:
release = releases[0]
plexpy.LATEST_RELEASE = release['tag_name']
if notify:
plexpy.NOTIFY_QUEUE.put({
'notify_action':
'on_plexpyupdate',
'plexpy_download_info':
release,
'plexpy_update_commit':
plexpy.LATEST_VERSION,
'plexpy_update_behind':
plexpy.COMMITS_BEHIND
})
if scheduler and plexpy.CONFIG.PLEXPY_AUTO_UPDATE and not plexpy.DOCKER and not plexpy.FROZEN:
logger.info('Running automatic update.')
plexpy.shutdown(restart=True, update=True)
elif plexpy.COMMITS_BEHIND == 0:
logger.info('Tautulli is up to date')
return plexpy.LATEST_VERSION
def dump_input(self):
if self.verbose < logger.INFO:
return self
logger.info(self, '')
logger.info(self, '******** %s flags ********', self.__class__)
logger.info(self, '* General Info')
logger.info(self, 'Date %s' % time.ctime())
logger.info(self, 'Python %s' % sys.version)
logger.info(self, 'Numpy %s' % numpy.__version__)
logger.info(self, 'Number of threads %d' % self.nthreads)
logger.info(self, 'Verbose level %d' % self.verbose)
logger.info(self, 'Scratch dir %s' % self.scratch)
logger.info(self, 'Correlated calculation %s' % self.corr)
logger.info(self, 'Input wfn data file %s' % self.wfnfile)
logger.info(self, 'Output h5 data file %s' % self.chkfile)
if (self.corr):
logger.info(self, '1-RDM data file %s' % self.rdmfile)
logger.info(self, 'Max_memory %d MB' % self.max_memory)
logger.info(self, '* Molecular Info')
logger.info(self, 'Num atoms %d' % self.natm)
logger.info(self, 'Num electrons %d' % self.nelectrons)
#logger.info(self,'Total charge %d' % self.charge)
#logger.info(self,'Spin %d ' % self.spin)
logger.info(self, 'Atom Coordinates (Bohr)')
for i in range(self.natm):
logger.info(
self,
'Nuclei %d %s with charge %d position : %.6f %.6f %.6f', i,
self.symbols[i], self.charges[i], *self.coords[i])
logger.info(self, '* Basis Info')
logger.info(self, 'Cuttoff for primitives %g' % self.cuttz)
logger.info(self, 'Number of Orbitals %d' % self.nmo)
logger.info(self, 'Old Number of primitives %d' % self.oldnprims)
logger.info(self, 'Number of primitives %d' % self.nprims)
logger.info(self, 'Maximum l in the basis %d' % self.lmax)
logger.debug(self, 'Number of primitives per center %s' % self.npc)
logger.info(self, 'Total number of shells %d' % self.nshells)
logger.debug(self, 'Number of shells per center %s' % self.ngroup)
for ic in range(self.natm):
logger.debug(self, 'Basis for center %d' % ic)
ntfu = numpy.zeros(6, dtype=numpy.int32)
for m in range(self.ngroup[ic]):
zz = self.oexp[self.nuexp[m, 0, ic] - 1]
ii = self.nuexp[m, 0, ic]
itip = self.ityp[ii - 1] - 1
it1 = data.nlm[itip, 0]
it2 = data.nlm[itip, 1]
it3 = data.nlm[itip, 2]
isu = it1 + it2 + it3
ntfu[isu] += 1
nzicm = self.nzexp[m, ic]
x1 = self.rcutte[m, ic]
logger.debug(self, 'Shell Type %s exp %f zero at %f idx %s' % \
(param.LTYPE[isu],zz,numpy.sqrt(x1),self.nuexp[m, :nzicm, ic]))
logger.debug(self, 'Number of shells of each type %s' % ntfu)
logger.debug(self, 'Ocupation of molecular orbitals %s' % self.mo_occ)
logger.info(self, '')
return self
def makeReport(i__UCMDBService, i__dir, i__FileName, queryTQL):
logger.info('************* START makeReport *************')
l__QueryService = i__UCMDBService.getTopologyQueryService()
#The unique name of the query node of type "node"
#l__NodeName = "Node"
#Creating a query node from type "host_node" and asking for all returned nodes the display_label
#Execute the unsaved query
## l__Topology = l__QueryService.executeQuery(l__QueryDefinition)
l__Topology = l__QueryService.executeNamedQuery(queryTQL)
#Get the node results
l__NodeCollection = l__Topology.getCIsByName("Node")
logger.debug(queryTQL + ' : ' + str(l__NodeCollection))
#Go over the nodes and print its related IPs
l__NodeQty = 0
#l__new = sorted(l__NodeCollection,key=attrgetter('label'))
#l__new = sorted(l__NodeCollection,key=lambda l__NodeCollection:l__NodeCollection[2])
#sorted(l__NodeCollection, key=itemgetter(2))
#logger.debug("\\\\\\\\\\\\\\\\"+l__NodeCollection.toString())
if l__NodeCollection:
newCI = ITAMline()
toprow = newCI.exportHeader("UDALL")
logger.debug("toprow : " + toprow)
output = ''
filename = newCI.exportfileName("UDALL")
file = openWrite(i__dir, filename)
if toprow != '':
output = toprow + '\r\n'
writeLine(file, output)
chunks = {}
chunkcount = 0
chunksize = 100
#citlist = ["host_node","unix","nt"]
for l__CITypeNode in l__NodeCollection:
if l__CITypeNode:
logger.debug(l__CITypeNode)
servername = str(l__CITypeNode.getPropertyValue("name"))
serverid = str(l__CITypeNode.getPropertyValue("global_id"))
serverdomain = str(l__CITypeNode.getPropertyValue("domain_name"))
#serverscannedtime = str(l__CITypeNode.getPropertyValue("root_lastaccesstime"))
servermodel = str(l__CITypeNode.getPropertyValue("node_model"))
servervendor = str(l__CITypeNode.getPropertyValue("vendor"))
serverserialnumber = str(
l__CITypeNode.getPropertyValue("serial_number"))
serverOS = str(
l__CITypeNode.getPropertyValue("discovered_os_name"))
logger.debug(servername)
intnum = 999
if ((chunkcount % chunksize) == 0):
chunks[(int(chunkcount / chunksize))] = []
chunks[(int(chunkcount / chunksize))].append(serverid)
chunkcount = chunkcount + 1
if (chunkcount > 0):
for id in chunks:
# retry the query up to 100 times
idlist = chunks[id]
logger.debug('ITAM %s : %s' % (id, idlist))
max_retry = 100
while (max_retry > 0):
try:
# Base query to get software for a node which will be dynamically run each time
logger.debug("execyqyery")
l__QueryExec = l__QueryService.createExecutableQuery(
"UD Node Software")
l__QueryExec.nodeRestrictions(
"Node").restrictToIdsFromStrings(idlist)
l__Topologysw = l__QueryService.executeQuery(l__QueryExec)
l__NodeCollection2 = l__Topologysw.getCIsByName("Node")
logger.debug(l__NodeCollection2)
max_retry = 0
for swnode in l__NodeCollection2:
logger.debug(swnode)
logger.debug("getall nnode related cis")
servername = removeExtra(
swnode.getPropertyValue("name"))
serverid = removeExtra(
swnode.getPropertyValue("global_id"))
serverdomain = removeExtra(
swnode.getPropertyValue("domain_name"))
serverscannedtime = getUDAScantime(swnode)
servermodel = removeExtra(
swnode.getPropertyValue("node_model"))
servervendor = removeExtra(
swnode.getPropertyValue("vendor"))
serverserialnumber = removeExtra(
swnode.getPropertyValue("serial_number"))
serverOS = removeExtra(
swnode.getPropertyValue("discovered_os_name"))
logger.debug(servername)
cpudata = getCPUsummary(swnode)
for l__TopologyRelation in swnode.getOutgoingRelations(
):
relcitype = removeExtra(
str(l__TopologyRelation.getEnd2CI().
getPropertyValue("root_class")))
if relcitype == 'installed_software':
logger.debug("in mainloop")
newCI = ITAMline()
newCI.devicelabel = servername
newCI.deviceid = serverid
newCI.softwarevendor = removeExtra(
l__TopologyRelation.getEnd2CI(
).getPropertyValue("discovered_vendor"))
newCI.softwarename = removeExtra(
l__TopologyRelation.getEnd2CI(
).getPropertyValue("name"))
newCI.releasename = removeExtra(
l__TopologyRelation.getEnd2CI(
).getPropertyValue("release"))
newCI.productversion = removeExtra(
l__TopologyRelation.getEnd2CI(
).getPropertyValue("version"))
newCI.domain = serverdomain
newCI.lastscannedtime = serverscannedtime
newCI.model = servermodel
newCI.servercompany = servervendor
newCI.serialnumber = serverserialnumber
newCI.installedpath = removeExtra(
l__TopologyRelation.getEnd2CI(
).getPropertyValue("file_system_path"))
newCI.OS = serverOS
newCI.physicalcpus = cpudata.physicalcpus
newCI.socketcount = cpudata.sockets
newCI.corecount = cpudata.corecount
newCI.cputype = cpudata.cputype
newCI.cpuvendor = cpudata.cpuvendor
newCI.logicalcpus = cpudata.logicalcpucount
newCI.cpuspeed = cpudata.cpuspeed
outstr = newCI.exportData("UDALL")
if outstr != '':
logger.debug("datathere")
writeLine(file, outstr + '\r\n')
except Exception as e:
logger.debug(
'UCMDB query for %s failed with %s, pausing for 5 seconds and retrying (count = %d)'
% (servername, e, max_retry))
time.sleep(5)
max_retry = max_retry - 1
if (max_retry == 0):
logger.debug('Abandoning query attempts')
#outputlist.append(newCI)
#logger.debug(servername + ' : ' + newCI.applicationname)
filename = filename[:-3] + 'success'
fileok = openWrite(i__dir, filename)
writeLine(fileok, 'Report generated successfully')
logger.info('************* END makeReport *************')
def readPropertiesFile(fname):
"""
Uses ConfigReader to read all sections in to a dictionary. Each sections options will
be kept as nested dictionary under each section key.
e.g.:
{
'WHITELIST': {
'<j2eetype-1>': {
'<value-1>',
'<value-2>',
:
'<value-n>'
}
'<j2eetype-2>': {
'<value-1>',
'<value-2>',
:
'<value-n>'
}
}
"""
l.debug("looking for file: '%s'", fname)
if not os.path.isfile(fname):
l.debug("file not found: '%s'", fname)
return {}
# discover comments per regex
re_comment = re.compile('^\s*#', re.IGNORECASE)
# discover enumeration on first level: eg "DATASOURCE3"
re_enumeration = re.compile('^([A-z]+)(\d+)\.', re.IGNORECASE)
reader = ConfigParser.ConfigParser()
l.debug("reading file: '%s'", fname)
reader.read(fname)
# read all sections and items therein
allSectionsMap = {}
sectionNames = reader.sections()
sectionNames.sort()
for sectionName in sectionNames:
if l.isDebugEnabled():
l.logLF()
l.debug("found section === %s === ", sectionName)
sectionMap = {}
allSectionsMap[sectionName.upper()] = sectionMap
lastBlockNumber = -1
expectedBlockNumber = 0
configBlockNumber = 0
# read all option lines from current section into sectionMap dictionary
# eg: "datasource1.connectionpool.reaptime = 7"
sectionOptions = reader.options(sectionName)
# in Python 2.1 (WAS8.x) the option() lines of a configParser section may be unsorted!
# but I cant use default sort, because Datasource1, datasource10 will be sorted before Datasource2
if l.isHellLevelEnabled():
for optionKey in sectionOptions:
l.debug666("read original-ordered section/line: %-20s : %s",
sectionName, optionKey)
sortedSectionOptions = sorted(sectionOptions,
key=configReaderOptionSort)
if l.isHellLevelEnabled():
for optionKey in sortedSectionOptions:
l.debug666("read num.sorted section/line: %s :\t%s",
sectionName, optionKey)
for optionKey in sortedSectionOptions:
if (re_comment.search(optionKey)) is not None:
l.debug666("skipping comment: %s", optionKey)
continue
optionValue = reader.get(sectionName, optionKey)
# now fix the *1st* level of enumeration, if it is not done sequentially.
# eg. when somebody declares 20 datasources, but deletes the 3rd.
# This way, one does not need to re-enumerate all the
# other datasource entries in the config file.
match = re_enumeration.search(optionKey)
if (match) is not None:
configBlockNumber = int(match.group(2))
if configBlockNumber > lastBlockNumber:
# new block found -> increase block counter
if l.isDebugEnabled():
l.logLF()
l.debug("=== read new config block # %d ===",
configBlockNumber)
expectedBlockNumber = expectedBlockNumber + 1
lastBlockNumber = configBlockNumber
l.debug2("config #:%2d expected #:%2d", configBlockNumber,
expectedBlockNumber)
# check for non-sequential block numbering!
if configBlockNumber != expectedBlockNumber:
l.debug("FIX block numbering: %d -> %d in option: '%s'",
configBlockNumber, expectedBlockNumber, optionKey)
optionKey = re_enumeration.sub(
lambda m, num=expectedBlockNumber: "%s%s." %
(m.group(1), num),
optionKey)
# and store the original block number in special hash key:
originalNumberKey = "%s%d.%s" % (match.group(1),
expectedBlockNumber,
"__ORIGINAL_NUMBER")
sectionMap[originalNumberKey.upper()] = configBlockNumber
l.debug("data: %s = %s", optionKey, optionValue)
# finally, add key to sectionMap hash:
sectionMap[optionKey.upper()] = optionValue
return allSectionsMap
def import_from_plexwatch(database=None,
table_name=None,
import_ignore_interval=0):
try:
connection = sqlite3.connect(database, timeout=20)
connection.row_factory = sqlite3.Row
except sqlite3.OperationalError:
logger.error(u"Tautulli Importer :: Invalid filename.")
return None
except ValueError:
logger.error(u"Tautulli Importer :: Invalid filename.")
return None
try:
connection.execute('SELECT ratingKey from %s' % table_name)
except sqlite3.OperationalError:
logger.error(
u"Tautulli Importer :: Database specified does not contain the required fields."
)
return None
logger.debug(u"Tautulli Importer :: PlexWatch data import in progress...")
ap = activity_processor.ActivityProcessor()
user_data = users.Users()
# Get the latest friends list so we can pull user id's
try:
users.refresh_users()
except:
logger.debug(
u"Tautulli Importer :: Unable to refresh the users list. Aborting import."
)
return None
query = 'SELECT time AS started, ' \
'stopped, ' \
'cast(ratingKey as text) AS rating_key, ' \
'null AS user_id, ' \
'user, ' \
'ip_address, ' \
'paused_counter, ' \
'platform AS player, ' \
'null AS platform, ' \
'null as machine_id, ' \
'parentRatingKey as parent_rating_key, ' \
'grandparentRatingKey as grandparent_rating_key, ' \
'null AS media_type, ' \
'null AS view_offset, ' \
'xml, ' \
'rating as content_rating,' \
'summary,' \
'title AS full_title,' \
'(case when orig_title_ep = "" then orig_title else ' \
'orig_title_ep end) as title,' \
'(case when orig_title_ep != "" then orig_title else ' \
'null end) as grandparent_title ' \
'FROM ' + table_name + ' ORDER BY id'
result = connection.execute(query)
for row in result:
# Extract the xml from the Plexwatch db xml field.
extracted_xml = extract_plexwatch_xml(row['xml'])
# If we get back None from our xml extractor skip over the record and log error.
if not extracted_xml:
logger.error(
u"Tautulli Importer :: Skipping record with ratingKey %s due to malformed xml."
% str(row['rating_key']))
continue
# Skip line if we don't have a ratingKey to work with
if not row['rating_key']:
logger.error(
u"Tautulli Importer :: Skipping record due to null ratingKey.")
continue
# If the user_id no longer exists in the friends list, pull it from the xml.
if user_data.get_user_id(user=row['user']):
user_id = user_data.get_user_id(user=row['user'])
else:
user_id = extracted_xml['user_id']
session_history = {
'started':
row['started'],
'stopped':
row['stopped'],
'rating_key':
row['rating_key'],
'title':
row['title'],
'parent_title':
extracted_xml['parent_title'],
'grandparent_title':
row['grandparent_title'],
'original_title':
extracted_xml['original_title'],
'full_title':
row['full_title'],
'user_id':
user_id,
'user':
row['user'],
'ip_address':
row['ip_address']
if row['ip_address'] else extracted_xml['ip_address'],
'paused_counter':
row['paused_counter'],
'player':
row['player'],
'platform':
extracted_xml['platform'],
'machine_id':
extracted_xml['machine_id'],
'parent_rating_key':
row['parent_rating_key'],
'grandparent_rating_key':
row['grandparent_rating_key'],
'media_type':
extracted_xml['media_type'],
'view_offset':
extracted_xml['view_offset'],
'video_decision':
extracted_xml['video_decision'],
'audio_decision':
extracted_xml['audio_decision'],
'transcode_decision':
extracted_xml['transcode_decision'],
'duration':
extracted_xml['duration'],
'width':
extracted_xml['width'],
'height':
extracted_xml['height'],
'container':
extracted_xml['container'],
'video_codec':
extracted_xml['video_codec'],
'audio_codec':
extracted_xml['audio_codec'],
'bitrate':
extracted_xml['bitrate'],
'video_resolution':
extracted_xml['video_resolution'],
'video_framerate':
extracted_xml['video_framerate'],
'aspect_ratio':
extracted_xml['aspect_ratio'],
'audio_channels':
extracted_xml['audio_channels'],
'transcode_protocol':
extracted_xml['transcode_protocol'],
'transcode_container':
extracted_xml['transcode_container'],
'transcode_video_codec':
extracted_xml['transcode_video_codec'],
'transcode_audio_codec':
extracted_xml['transcode_audio_codec'],
'transcode_audio_channels':
extracted_xml['transcode_audio_channels'],
'transcode_width':
extracted_xml['transcode_width'],
'transcode_height':
extracted_xml['transcode_height']
}
session_history_metadata = {
'rating_key': helpers.latinToAscii(row['rating_key']),
'parent_rating_key': row['parent_rating_key'],
'grandparent_rating_key': row['grandparent_rating_key'],
'title': row['title'],
'parent_title': extracted_xml['parent_title'],
'grandparent_title': row['grandparent_title'],
'original_title': extracted_xml['original_title'],
'media_index': extracted_xml['media_index'],
'parent_media_index': extracted_xml['parent_media_index'],
'thumb': extracted_xml['thumb'],
'parent_thumb': extracted_xml['parent_thumb'],
'grandparent_thumb': extracted_xml['grandparent_thumb'],
'art': extracted_xml['art'],
'media_type': extracted_xml['media_type'],
'year': extracted_xml['year'],
'originally_available_at':
extracted_xml['originally_available_at'],
'added_at': extracted_xml['added_at'],
'updated_at': extracted_xml['updated_at'],
'last_viewed_at': extracted_xml['last_viewed_at'],
'content_rating': row['content_rating'],
'summary': row['summary'],
'tagline': extracted_xml['tagline'],
'rating': extracted_xml['rating'],
'duration': extracted_xml['duration'],
'guid': extracted_xml['guid'],
'section_id': extracted_xml['section_id'],
'directors': extracted_xml['directors'],
'writers': extracted_xml['writers'],
'actors': extracted_xml['actors'],
'genres': extracted_xml['genres'],
'studio': extracted_xml['studio'],
'labels': extracted_xml['labels'],
'full_title': row['full_title'],
'width': extracted_xml['width'],
'height': extracted_xml['height'],
'container': extracted_xml['container'],
'video_codec': extracted_xml['video_codec'],
'audio_codec': extracted_xml['audio_codec'],
'bitrate': extracted_xml['bitrate'],
'video_resolution': extracted_xml['video_resolution'],
'video_framerate': extracted_xml['video_framerate'],
'aspect_ratio': extracted_xml['aspect_ratio'],
'audio_channels': extracted_xml['audio_channels']
}
# On older versions of PMS, "clip" items were still classified as "movie" and had bad ratingKey values
# Just make sure that the ratingKey is indeed an integer
if session_history_metadata['rating_key'].isdigit():
ap.write_session_history(
session=session_history,
import_metadata=session_history_metadata,
is_import=True,
import_ignore_interval=import_ignore_interval)
else:
logger.debug(u"Tautulli Importer :: Item has bad rating_key: %s" %
session_history_metadata['rating_key'])
logger.debug(u"Tautulli Importer :: PlexWatch data import complete.")
import_users()
def DiscoveryMain(Framework):
errMsg = "Export Directory is not valid. Ensure export directory exists on the probe system."
testConnection = Framework.getTriggerCIData("testConnection")
if testConnection == 'true':
# check if valid export directory exists
isValid = validateDirectory(Framework)
if not isValid:
raise Exception, errMsg
return
else:
logger.debug("Test connection was successful")
return
i__Framework = Framework
logger.info('************* START MAIN *************')
l__LocalShell = shellutils.ShellUtils(
i__Framework.createClient(ClientsConsts.LOCAL_SHELL_PROTOCOL_NAME))
l__OSHVResult = ObjectStateHolderVector()
l__UCMDBService = createCMDBConnection(l__LocalShell)
logger.debug(str(l__UCMDBService))
if not validateDirectory(Framework):
logger.error(errMsg)
raise Exception, errMsg
return
expDirPath = Framework.getTriggerCIData("Export Directory")
queryTQL = Framework.getTriggerCIData("TQL Input Name")
isLastChunk = Framework.getTriggerCIData("isLastChunk")
logger.debug("Generating ITAM report output")
# clean up XML
empty = 0 ##isEmpty(addResult, "addResult")
if not empty:
##addResult = replace(addResult, iplookup)
if l__UCMDBService:
logger.debug("Calling makereport")
#addResult = makeReport(l__UCMDBService, expDirPath, 'testout.txt', queryTQL)
makeReport(l__UCMDBService, expDirPath, 'testout.txt', queryTQL)
logger.debug("Back from makereport")
reports = [] #'ITAMALL']
addResult = 0
if addResult:
for rep in reports:
toprow = addResult[0].exportHeader(rep)
logger.debug("report : " + rep)
logger.debug("toprow : " + toprow)
output = ''
try:
filename = addResult[0].exportfileName(rep)
file = openWrite(expDirPath, filename)
if toprow != '':
output = toprow + '\r\n'
writeLine(file, output)
for ddmi in addResult:
outstr = ddmi.exportData(rep)
#logger.debug('out: ' + ddmi.exportData(rep))
if outstr != '':
#output = output + outstr + '\r\n'
writeLine(file, outstr + '\r\n')
except:
logger.debug("Error writing to file for report " + rep +
" filename - " + expDirPath + "/" +
addResult[0].exportfileName(rep))
def verify_directories_created():
import logger
import os
logger.info("verify_directories_created")
# Force download path if empty
download_path = get_setting("downloadpath")
if download_path == "":
download_path = os.path.join(get_data_path(), "downloads")
set_setting("downloadpath", download_path)
# Force download list path if empty
download_list_path = get_setting("downloadlistpath")
if download_list_path == "":
download_list_path = os.path.join(get_data_path(), "downloads", "list")
set_setting("downloadlistpath", download_list_path)
# Force bookmark path if empty
bookmark_path = get_setting("bookmarkpath")
if bookmark_path == "":
bookmark_path = os.path.join(get_data_path(), "bookmarks")
set_setting("bookmarkpath", bookmark_path)
# Create data_path if not exists
if not os.path.exists(get_data_path()):
logger.debug("Creating data_path " + get_data_path())
try:
os.mkdir(get_data_path())
except:
pass
# Create download_path if not exists
if not download_path.lower().startswith("smb") and not os.path.exists(
download_path):
logger.debug("Creating download_path " + download_path)
try:
os.mkdir(download_path)
except:
pass
# Create download_list_path if not exists
if not download_list_path.lower().startswith("smb") and not os.path.exists(
download_list_path):
logger.debug("Creating download_list_path " + download_list_path)
try:
os.mkdir(download_list_path)
except:
pass
# Create bookmark_path if not exists
if not bookmark_path.lower().startswith("smb") and not os.path.exists(
bookmark_path):
logger.debug("Creating bookmark_path " + bookmark_path)
try:
os.mkdir(bookmark_path)
except:
pass
# Create library_path if not exists
if not get_library_path().lower().startswith("smb") and not os.path.exists(
get_library_path()):
logger.debug("Creating library_path " + get_library_path())
try:
os.mkdir(get_library_path())
except:
pass
def get_plex_downloads(self):
logger.debug(u"Tautulli PlexTV :: Retrieving current server version.")
pms_connect = pmsconnect.PmsConnect()
pms_connect.set_server_version()
update_channel = pms_connect.get_server_update_channel()
logger.debug(u"Tautulli PlexTV :: Plex update channel is %s." %
update_channel)
plex_downloads = self.get_plextv_downloads(
plexpass=(update_channel == 'beta'))
try:
available_downloads = json.loads(plex_downloads)
except Exception as e:
logger.warn(
u"Tautulli PlexTV :: Unable to load JSON for get_plex_updates."
)
return {}
# Get the updates for the platform
pms_platform = common.PMS_PLATFORM_NAME_OVERRIDES.get(
plexpy.CONFIG.PMS_PLATFORM, plexpy.CONFIG.PMS_PLATFORM)
platform_downloads = available_downloads.get('computer').get(pms_platform) or \
available_downloads.get('nas').get(pms_platform)
if not platform_downloads:
logger.error(
u"Tautulli PlexTV :: Unable to retrieve Plex updates: Could not match server platform: %s."
% pms_platform)
return {}
v_old = helpers.cast_to_int("".join(
v.zfill(4)
for v in plexpy.CONFIG.PMS_VERSION.split('-')[0].split('.')[:4]))
v_new = helpers.cast_to_int("".join(
v.zfill(4) for v in platform_downloads.get('version', '').split(
'-')[0].split('.')[:4]))
if not v_old:
logger.error(
u"Tautulli PlexTV :: Unable to retrieve Plex updates: Invalid current server version: %s."
% plexpy.CONFIG.PMS_VERSION)
return {}
if not v_new:
logger.error(
u"Tautulli PlexTV :: Unable to retrieve Plex updates: Invalid new server version: %s."
% platform_downloads.get('version'))
return {}
# Get proper download
releases = platform_downloads.get('releases', [{}])
release = next(
(r for r in releases
if r['distro'] == plexpy.CONFIG.PMS_UPDATE_DISTRO
and r['build'] == plexpy.CONFIG.PMS_UPDATE_DISTRO_BUILD),
releases[0])
download_info = {
'update_available': v_new > v_old,
'platform': platform_downloads.get('name'),
'release_date': platform_downloads.get('release_date'),
'version': platform_downloads.get('version'),
'requirements': platform_downloads.get('requirements'),
'extra_info': platform_downloads.get('extra_info'),
'changelog_added': platform_downloads.get('items_added'),
'changelog_fixed': platform_downloads.get('items_fixed'),
'label': release.get('label'),
'distro': release.get('distro'),
'distro_build': release.get('build'),
'download_url': release.get('url'),
}
return download_info
def adjust_ahrs_trim(conn, level_attitude):
'''
force the AHRS trim to zero, which removes the effect of the jig not being quite
level. This is only incorrect if the accels are not aligned on the board, which
we check in check_accel_cal later
As a side effect this function set AHRS_ORIENTATION for the test board to 12
'''
# start with board right way up
rotate.set_rotation(conn, 'level')
# we need to work out what the error in attitude of the 3 IMUs on the test jig is
# to do that we start with it level, and measure the roll/pitch as compared to the reference
conn.discard_messages()
ref_imu = conn.refmav.recv_match(type='RAW_IMU', blocking=True, timeout=3)
test_imu1 = conn.testmav.recv_match(type='RAW_IMU',
blocking=True,
timeout=3)
test_imu2 = conn.testmav.recv_match(type='SCALED_IMU2',
blocking=True,
timeout=3)
test_imu3 = conn.testmav.recv_match(type='SCALED_IMU3',
blocking=True,
timeout=3)
if ref_imu is None:
util.failure("Lost comms to reference board in ahrs trim")
if test_imu1 is None or test_imu2 is None or test_imu3 is None:
util.failure("Lost comms to test board in ahrs trim")
(ref_roll, ref_pitch) = util.attitude_estimate(ref_imu)
(test_roll1, test_pitch1) = util.attitude_estimate(test_imu1)
(test_roll2, test_pitch2) = util.attitude_estimate(test_imu2)
(test_roll3, test_pitch3) = util.attitude_estimate(test_imu3)
# get the roll and pitch errors
roll_error1 = (test_roll1 - ref_roll)
roll_error2 = (test_roll2 - ref_roll)
roll_error3 = (test_roll3 - ref_roll)
pitch_error1 = (test_pitch1 - ref_pitch)
pitch_error2 = (test_pitch2 - ref_pitch)
pitch_error3 = (test_pitch3 - ref_pitch)
conn.discard_messages()
ref_imu = conn.refmav.recv_match(type='RAW_IMU', blocking=True, timeout=3)
test_imu1 = conn.testmav.recv_match(type='RAW_IMU',
blocking=True,
timeout=3)
test_imu2 = conn.testmav.recv_match(type='SCALED_IMU2',
blocking=True,
timeout=3)
test_imu3 = conn.testmav.recv_match(type='SCALED_IMU3',
blocking=True,
timeout=3)
if ref_imu is None:
util.failure("Lost comms to reference board in ahrs trim")
if test_imu1 is None or test_imu2 is None or test_imu3 is None:
util.failure("Lost comms to test board in ahrs trim")
logger.debug(
"Tilt Ref=(%.1f %.1f) Test1=(%.1f %.1f) Test2=(%.1f %.1f) Test3=(%.1f %.1f)"
% (ref_roll, ref_pitch, test_roll1, test_pitch1, test_roll2,
test_pitch2, test_roll3, test_pitch3))
if (abs(ref_roll) > ROTATION_TOLERANCE
or abs(ref_pitch) > ROTATION_TOLERANCE):
util.failure("Reference board rotation error")
logger.debug("Tilt offsets: Roll(%.1f %.1f %.1f) Pitch(%.1f %.1f %.1f) " %
(roll_error1, roll_error2, roll_error3, pitch_error1,
pitch_error2, pitch_error3))
if (abs(roll_error1) > TILT_TOLERANCE1
or abs(roll_error2) > TILT_TOLERANCE1
or abs(roll_error3) > TILT_TOLERANCE3):
util.failure("Test board roll error")
if (abs(pitch_error1) > TILT_TOLERANCE1
or abs(pitch_error2) > TILT_TOLERANCE1
or abs(pitch_error3) > TILT_TOLERANCE3):
util.failure("Test board pitch error")
# flip upside down for the trim calculation
rotate.set_rotation(conn, 'back')
# set orientation upside down for trim measurement
util.param_set(conn.test, 'AHRS_ORIENTATION', 12)
# sleep an extra four seconds - we need to be very sure the board is still for trim
time.sleep(4)
conn.discard_messages()
# average over 30 samples for trim
num_samples = 30
test_roll = [0] * 3
test_pitch = [0] * 3
msgs = ['RAW_IMU', 'SCALED_IMU2', 'SCALED_IMU3']
for i in range(num_samples):
for j in range(3):
test_imu = conn.testmav.recv_match(type=msgs[j],
blocking=True,
timeout=3)
if test_imu is None:
util.failure("Lost comms to test board in ahrs trim")
(roll, pitch) = util.attitude_estimate(test_imu)
test_roll[j] += roll
test_pitch[j] += pitch
for j in range(3):
test_roll[j] /= num_samples
test_pitch[j] /= num_samples
logger.debug(
"Average Trim tilt Test1=(%.1f %.1f) Test2=(%.1f %.1f) Test3=(%.1f %.1f)"
% (test_roll[0], test_pitch[0], test_roll[1], test_pitch[1],
test_roll[2], test_pitch[2]))
# setting a positive trim value reduces the attitude that is
# read. So setting a trim of 0.1 when level results in a attitude
# reading of -5.8 degrees
# this approach assumes the mpu6000 on the FMU (IMU3) is level
# with respect to the board, and that any attitude error is due to
# the isolation board mount. We use the average of the error from
# IMU1 and IMU2
trim_x = radians((test_roll[0] + test_roll[1]) * 0.5 - test_roll[2])
trim_y = radians((test_pitch[0] + test_pitch[1]) * 0.5 - test_pitch[2])
util.param_set(conn.test, 'AHRS_TRIM_X', trim_x)
time.sleep(0.2)
util.param_set(conn.test, 'AHRS_TRIM_Y', trim_y)
time.sleep(0.2)
logger.debug("Set trims AHRS_TRIM_X=%.4f AHRS_TRIM_Y=%.4f" %
(trim_x, trim_y))
def processManuallyEnrichedScanFile(Framework):
files = File(
CollectorsParameters.PROBE_MGR_INVENTORY_XMLENRICHER_FILES_FOLDER +
XmlEnricherConstants.SENDING_FOLDER_NAME).listFiles(XsfFilter())
if not files or not len(files):
logger.debug('no manually file found!')
Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
return
filesNum = len(files)
count = 0
noFileProcessedSuccess = filesNum
while count < filesNum:
file = files[count]
count += 1
path = file.getAbsolutePath()
try:
logger.debug(file.getAbsolutePath())
ParseEnrichedScanFile.parseFile(Framework,
path,
isManual=1,
reportWarning=1)
if Framework.getStepExecutionStatus(
) == WorkflowStepStatus.SUCCESS:
noFileProcessedSuccess = 0
except:
errorMessage = str(sys.exc_info()[1])
logger.debug('Failed parsing file: ' + path)
logger.debugException(errorMessage)
Framework.reportWarning(
inventoryerrorcodes.INVENTORY_DISCOVERY_FAILED_PARSING, [path])
logger.debug(noFileProcessedSuccess)
if noFileProcessedSuccess:
logger.debug('All Scan file(s) processed failed')
Framework.setStepExecutionStatus(WorkflowStepStatus.FAILURE)
Framework.reportError(
inventoryerrorcodes.INVENTORY_DISCOVERY_FAILED_PARSING,
['All Scan file(s) processed failed'])
else:
logger.debug(
'OK, due to some files successed process, set the status as SUCCESS'
)
Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
