예제 #1
0
파일: parser.py 프로젝트: Seliverstoff/cgru
	def parse( self, data, mode):
		
		data = cgruutils.toStr( data)
		mode = cgruutils.toStr( mode)

		self.activity = ''
		self.warning = False
		self.error = False
		self.badresult = False
		result = None
		if data.find( str_warning         ) != -1: self.warning         = True
		if data.find( str_error           ) != -1: self.error           = True
		if data.find( str_badresult       ) != -1: self.badresult       = True
		if data.find( str_finishedsuccess ) != -1: self.finishedsuccess = True

		lines = data.split('\n')
		for line in lines:
			if line.find('@IMAGE@') != -1:
				line = line[7:]
				self.appendFile( line.strip())

		try:
			result = self.do( data, mode)
		except:
			print('Error parsing output:')
			print( str(sys.exc_info()[1]))
		return result, self.percent, self.frame, self.percentframe, self.warning, self.error, self.badresult, self.finishedsuccess, self.activity
예제 #2
0
    def generateThumbnail(self):
        """Missing DocString

		:return:
		"""
        cmds = []

        if not os.path.isdir(self.taskInfo['store_dir']):
            return cmds

        files_list = []
        if self.parser is not None:
            files_list = self.parser.getFiles()

        if len(files_list):
            if len(files_list) > 3:
                files_list = [
                    files_list[0], files_list[int(len(files_list) / 2)],
                    files_list[-1]
                ]
        elif len(self.taskInfo['files']):
            for afile in self.taskInfo['files']:
                files_list.append(afile)
            # files_list.append(afile.decode('utf-8'))
        else:
            return cmds

        for image in files_list:
            image = cgruutils.toStr(image)
            if len(image) < 1:
                continue
            image = os.path.join(self.taskInfo['wdir'], image)
            if not os.path.isfile(image):
                continue

            basename, ext = os.path.splitext(os.path.basename(image))
            if len(ext) < 2:
                continue
            ext = ext.lower()[1:]
            if not ext in cgruconfig.VARS['af_thumbnail_extensions']:
                continue

            store_dir = cgruutils.toStr(self.taskInfo['store_dir'])
            thumbnail = os.path.basename(image) + '.jpg'
            thumbnail = store_dir + '/' + thumbnail

            self.taskInfo['image'] = os.path.normpath(image)
            self.taskInfo['thumbnail'] = os.path.normpath(thumbnail)
            self.taskInfo['pre_args'] = ''
            if ext == 'dpx' or ext == 'cin':
                self.taskInfo['pre_args'] = '-set colorspace Log'
            if ext == 'exr':
                self.taskInfo['pre_args'] = '-set colorspace RGB'

            cmd = str(cgruconfig.VARS['af_thumbnail_cmd']) % self.taskInfo
            # print( cmd)
            # cmds.append('echo ' + cmd)
            cmds.append(cmd)

        return cmds
예제 #3
0
	def init(self, pathsmap, Verbose=False):
		"""Missing DocString

		:param pathsmap:
		:param Verbose:
		:return:
		"""
		self.initialized = False
		self.PathClient = []
		self.PathServer = []

		for pair in pathsmap:
			if len(pair) != 2:
				print('ERROR: Pathmap is not a pair.')
				return
			path_client = (cgruutils.toStr(pair[0])).replace('\\', '/')
			path_server = (cgruutils.toStr(pair[1])).replace('\\', '/')
			self.PathClient.append(path_client)
			self.PathServer.append(path_server)
			self.initialized = True

		if Verbose:
			print('Path map:')
			n = 0
			for path in self.PathClient:
				print('    "%s" <-> "%s"' % (path, self.PathServer[n]))
				n += 1
			if self.UnixSeparators:
				print('Using UNIX separators.')
예제 #4
0
    def processPattern(self, i_block, i_tasks, i_start, i_finish):
        """ Fill numbers for numeric blocks or combine block and tasks patterns for blocks with tasks
        :param i_block: a string with a block part
        :param i_tasks: an array of strings with a tasks parts for a not numeric blocks
        :retutn: as array combined of stings (1 element in array for a numeric)
        """
        block = cgruutils.toStr(i_block)

        if self.numeric:
            return [afcommon.fillNumbers(block, i_start, i_finish)]

        array = []
        for task in i_tasks:
            if len(task) == 0:
                continue
            task = cgruutils.toStr(task)
            result = block
            if len(result):
                if result.find('@#@') != -1:
                    result = result.replace('@#@', task)
                else:
                    result += task
            else:
                result = task
            array.append(result)

        if len(array) == 0:
            array.append(block)

        return array
예제 #5
0
    def parse(self, data, mode):
        """Missing DocString

		:param data:
		:param mode:
		:return:
		"""

        data = cgruutils.toStr(data)
        mode = cgruutils.toStr(mode)

        self.activity = ""
        self.warning = False
        self.error = False
        self.badresult = False
        result = None
        if data.find(str_warning) != -1:
            self.warning = True
        if data.find(str_error) != -1:
            self.error = True
        if data.find(str_badresult) != -1:
            self.badresult = True
        if data.find(str_finishedsuccess) != -1:
            self.finishedsuccess = True

        lines = data.split("\n")
        for line in lines:
            if line.find("@IMAGE@") != -1:  # Will be used in CGRU render scripts
                line = line[7:]
                self.appendFile(line.strip(), False)
            if line.find("Image: ") == 0:  # ImageMagick
                line = line[7:]
                self.appendFile(line.strip(), False)
            if (
                line.find("@IMAGE!@") != -1
            ):  # Will be used in CGRU render scripts to generate thumb while task is still running
                line = line[8:]
                self.appendFile(line.strip(), True)

        try:
            result = self.do(data, mode)
        except:  # TODO: too broad exception clause
            print("Error parsing output:")
            # print(str(sys.exc_info()[1]))
            traceback.print_exc(file=sys.stdout)
        return (
            result,
            self.percent,
            self.frame,
            self.percentframe,
            self.warning,
            self.error,
            self.badresult,
            self.finishedsuccess,
            self.activity,
        )
예제 #6
0
    def translateFile(self, infile, outfile, toserver, SearchStrings, Verbose):
        if not self.initialized: return True
        if Verbose:
            print('TranslateFile:')
            print('Input file: "%s"' % infile)
            print('Output file: "%s"' % outfile)
        filein = open(infile, 'r')
        inlines = filein.readlines()
        filein.close()
        outdata = []
        for line in inlines:
            if not isinstance(line, str): line = cgruutils.toStr(line)
            toskip = False
            if len(SearchStrings): toskip = True
            for searchstr in SearchStrings:
                #				if line[0:len(searchstr)] == searchstr:
                if line.find(searchstr) != -1:
                    #					print('"%s" == "%s"' % (line[0:len(searchstr)], searchstr))
                    toskip = False
                    break
            if toskip:
                outdata.append(line)
            else:
                #				print('Translating: "%s"' % line)
                outdata.append(self.translatePath(line, toserver, Verbose))
#				print( outdata[-1])
        fileout = open(outfile, 'w')
        fileout.writelines(outdata)
        fileout.close()
        return True
예제 #7
0
	def translateFile( self, infile, outfile, toserver, SearchStrings, Verbose):
		if not self.initialized: return True
		if Verbose:
			print('TranslateFile:')
			print('Input file: "%s"' % infile)
			print('Output file: "%s"' % outfile)
		filein = open( infile, 'r')
		inlines = filein.readlines()
		filein.close()
		outdata = []
		for line in inlines:
			if not isinstance( line, str): line = cgruutils.toStr( line)
			toskip = False
			if len( SearchStrings): toskip = True
			for searchstr in SearchStrings:
#				if line[0:len(searchstr)] == searchstr:
				if line.find( searchstr) != -1:
#					print('"%s" == "%s"' % (line[0:len(searchstr)], searchstr))
					toskip = False
					break
			if toskip:
				outdata.append( line)
			else:
#				print('Translating: "%s"' % line)
				outdata.append( self.translatePath( line, toserver , Verbose))
#				print( outdata[-1])
		fileout = open( outfile, 'w')
		fileout.writelines(outdata)
		fileout.close()
		return True
예제 #8
0
	def translateFile( self, infile, outfile, toserver, SearchStrings, Verbose):
		if not self.initialized: return True
		if Verbose:
			print('TranslateFile:')
			print('Input file: "%s"' % infile)
			print('Output file: "%s"' % outfile)
		filein = open( infile, 'r')
		inlines = filein.readlines()
		filein.close()
		outdata = ''
		for line in inlines:
			if not isinstance( line, str): line = cgruutils.toStr( line)
			toskip = False
			if len( SearchStrings): toskip = True
			for searchstr in SearchStrings:
				if line.find( searchstr) != -1:
					toskip = False
					continue
			if toskip:
				lineout = line
			else:
				lineout = self.translatePath( line, toserver , Verbose)
			outdata += lineout + '\n'
		fileout = open( outfile, 'w')
		fileout.write(outdata)
		fileout.close()
		return True
예제 #9
0
파일: service.py 프로젝트: wildparky/cgru
	def generateThumbnail( self):
		cmds = []

		if not os.path.isdir( self.taskInfo['store_dir']):
			return cmds

		files_list = []
		if self.parser is not None:
			files_list = self.parser.getFiles()

		if len( files_list ):
			if len( files_list) > 3:
				files_list = [ files_list[0], files_list[ int(len(files_list)/2) ], files_list[-1]]
		elif len(self.taskInfo['files']):
			for afile in self.taskInfo['files']:
				files_list.append( afile)
				#files_list.append( afile.decode('utf-8'))
		else:
			return cmds

		for image in files_list:
			image = cgruutils.toStr( image)
			if len( image) < 1: continue
			image = os.path.join( self.taskInfo['wdir'], image)
			if not os.path.isfile( image): continue

			basename, ext = os.path.splitext( os.path.basename( image))
			if len( ext ) < 2: continue
			ext = ext.lower()[1:]
			if not ext in cgruconfig.VARS['af_thumbnail_extensions']: continue

			store_dir = cgruutils.toStr( self.taskInfo['store_dir'])
			thumbnail = os.path.basename( image) + '.jpg'
			thumbnail = store_dir + '/' + thumbnail

			self.taskInfo['image'] = os.path.normpath( image)
			self.taskInfo['thumbnail'] = os.path.normpath( thumbnail)
			self.taskInfo['pre_args'] = ''
			if ext == 'dpx' or ext == 'cin': self.taskInfo['pre_args'] = '-set colorspace Log'
			if ext == 'exr': self.taskInfo['pre_args'] = '-set colorspace RGB'

			cmd = str(cgruconfig.VARS['af_thumbnail_cmd']) % (self.taskInfo)
			#print( cmd)
			#cmds.append('echo ' + cmd)
			cmds.append( cmd)

		return cmds
예제 #10
0
    def readCommand(self):

        if self.qsocket is None:
            return

        print('Server reading...')

        cmd.execute(cgruutils.toStr(self.qsocket.readAll().data()))
예제 #11
0
파일: server.py 프로젝트: CGRU/cgru
    def readCommand(self):

        if self.qsocket is None:
            return

        print('Server reading...')

        cmd.execute( cgruutils.toStr( self.qsocket.readAll().data()))
예제 #12
0
파일: serverhttps.py 프로젝트: CGRU/cgru
    def do_POST( self):
        content_len = int(self.headers['content-length'])
        post_body = cgruutils.toStr( self.rfile.read(content_len))

        cmd.execute( post_body)
        #print('Executing:')
        #print( post_body)
        #subprocess.Popen( post_body, shell=True)

        self.writeResponse( b'STATUS: OK')
예제 #13
0
    def do_POST(self):
        content_len = int(self.headers['content-length'])
        post_body = cgruutils.toStr(self.rfile.read(content_len))

        cmd.execute(post_body)
        #print('Executing:')
        #print( post_body)
        #subprocess.Popen( post_body, shell=True)

        self.writeResponse(b'STATUS: OK')
예제 #14
0
파일: parser.py 프로젝트: undomick/cgru
	def parse(self, data, mode):
		"""Missing DocString

		:param data:
		:param mode:
		:return:
		"""

		data = cgruutils.toStr(data)
		mode = cgruutils.toStr(mode)

		self.activity = ''
		self.warning = False
		self.error = False
		self.badresult = False
		result = None
		if data.find(str_warning) != -1:
			self.warning = True
		if data.find(str_error) != -1:
			self.error = True
		if data.find(str_badresult) != -1:
			self.badresult = True
		if data.find(str_finishedsuccess) != -1:
			self.finishedsuccess = True

		lines = data.split('\n')
		for line in lines:
			if line.find('@IMAGE@') != -1: # Will be used in CGRU render scripts
				line = line[7:]
				self.appendFile(line.strip())
			if line.find('Image: ') == 0: # ImageMagick
				line = line[7:]
				self.appendFile(line.strip())

		try:
			result = self.do(data, mode)
		except:  # TODO: too broad exception clause
			print('Error parsing output:')
			#print(str(sys.exc_info()[1]))
			traceback.print_exc(file=sys.stdout)
		return result, self.percent, self.frame, self.percentframe, \
			   self.warning, self.error, self.badresult, self.finishedsuccess, \
			   self.activity
예제 #15
0
파일: mediainfo.py 프로젝트: CGRU/cgru
def processMovie(i_file):

    out = dict()

    out['infile'] = i_file

    if not os.path.isfile(out['infile']):
        out['error'] = 'Input file does not exist.'
        return out

    params = {}
    params['bitdepth'] = 'BitDepth'
    params['chromasubsampling'] = 'ChromaSubsampling'
    params['codec'] = 'Codec'
    params['colorspace'] = 'ColorSpace'
    params['fps'] = 'FrameRate'
    params['frame_count'] = 'FrameCount'
    params['width'] = 'Width'
    params['height'] = 'Height'

    inform = ''
    for key in params:
        if len(inform): inform += ','
        inform += '""%s"":""%%%s%%""' % (key, params[key])
    inform = '--Inform=Video;{' + inform + '}'

    data = subprocess.check_output(['mediainfo', inform, out['infile']])
    data = cgruutils.toStr(data)

    inform = None
    try:
        inform = json.loads(data)
    except:
        inform = None
        out['data'] = data
        out['error'] = 'json load error'
        print(traceback.format_exc())
        return out

    # remove empty keys:
    inform = dict([(k,v) for k,v in inform.items() if len(v)>0])

    for key in inform:
        if inform[key].isdigit():
            inform[key] = int(inform[key])
        else:
            try:
                inform[key] = float(inform[key])
            except:
                pass

    # return this object:
    out['mediainfo'] = {'video':inform}
    return out
예제 #16
0
    def parse(self, data, mode):
        """Missing DocString

        :param data:
        :param mode:
        :return:
        """

        data = cgruutils.toStr(data)
        mode = cgruutils.toStr(mode)

        if len(data):
            self.doBaseCheck(data, mode)

        try:
            self.result = self.do(data, mode)
        except:  # TODO: too broad exception clause
            print('Error parsing output:')
            # print(str(sys.exc_info()[1]))
            traceback.print_exc(file=sys.stdout)
예제 #17
0
def processExif(i_file):
    out = dict()

    out['infile'] = i_file

    if not os.path.isfile(out['infile']):
        out['error'] = 'Input file does not exist.'
        return out

    params = {}
    params['BitDepth'] = 'bitdepth'
    params['ColorType'] = 'colortype'
    params['Compression'] = 'compression'
    params['ImageWidth'] = 'width'
    params['ImageHeight'] = 'height'
    params['Artist'] = 'artist'
    params['Comment'] = 'comments'

    cmd_args = ['exiftool', '-S', '-fast2']
    for key in params:
        cmd_args.append('-' + key)
    cmd_args.append(i_file)

    try:
        data = subprocess.check_output(cmd_args)
    except:
        out['error'] = 'Failed to execute process.'
        return out

    data = cgruutils.toStr(data)
    data = data.splitlines()
    exif = dict()

    for line in data:
        if len(line) == 0: continue
        fields = line.split(': ', 1)
        if len(fields) != 2: continue
        key = fields[0]
        val = fields[1]
        if len(val) == 0: continue
        if not key in params: continue

        if val.isdigit(): val = int(val)
        else:
            try:
                val = float(val)
            except:
                pass

        exif[params[key]] = val

    # return this object:
    out['mediainfo'] = {'exif': exif}
    return out
예제 #18
0
    def expandEnvVars(self, i_str):

        if not 'environment' in self.taskInfo:
            return i_str

        for name in self.taskInfo['environment']:
            i_str = i_str.replace(
                '__%s__' % name,
                cgruutils.toStr(self.taskInfo['environment'][name]))

        return i_str
예제 #19
0
def processMovie(i_file):

    out = dict()

    out['infile'] = i_file

    if not os.path.isfile(out['infile']):
        out['error'] = 'Input file does not exist.'
        return out

    params = {}
    params['bitdepth'] = 'BitDepth'
    params['chromasubsampling'] = 'ChromaSubsampling'
    params['codec'] = 'Codec'
    params['colorspace'] = 'ColorSpace'
    params['fps'] = 'FrameRate'
    params['frame_count'] = 'FrameCount'
    params['width'] = 'Width'
    params['height'] = 'Height'

    inform = ''
    for key in params:
        if len(inform): inform += ','
        inform += '""%s"":""%%%s%%""' % (key, params[key])
    inform = '--Inform=Video;{' + inform + '}'

    data = subprocess.check_output(['mediainfo', inform, out['infile']])
    data = cgruutils.toStr(data)

    inform = None
    try:
        inform = json.loads(data)
    except:
        inform = None
        out['data'] = data
        out['error'] = 'json load error'
        print(traceback.format_exc())
        return out

    # remove empty keys:
    inform = dict([(k, v) for k, v in inform.items() if len(v) > 0])

    for key in inform:
        if inform[key].isdigit():
            inform[key] = int(inform[key])
        else:
            try:
                inform[key] = float(inform[key])
            except:
                pass

    # return this object:
    out['mediainfo'] = {'video': inform}
    return out
예제 #20
0
    def parse(self, i_args):
        """Missing DocString
        :return:
        """

        i_args['data'] = cgruutils.toStr(i_args['data'])
        if 'mode' in i_args:
            i_args['mode'] = cgruutils.toStr(i_args['mode'])
        if 'pid' in i_args:
            self.pid = i_args['pid']

        self.processResources(i_args)

        if len(i_args['data']):
            self.doBaseCheck(i_args['data'])
        try:
            self.result = self.do(i_args)
        except:  # TODO: too broad exception clause
            print('Error parsing output:')
            # print(str(sys.exc_info()[1]))
            traceback.print_exc(file=sys.stdout)
예제 #21
0
    def toHTML(self, i_data):
        """ Convert data to HTML.
            Designed for GUIs for escape sequences, errors highlighting.
        :param i_data: input data
        :return: converted data
        """
        lines = cgruutils.toStr(i_data).replace('\r', '').split('\n')
        html = []

        for line in lines:
            html.append(self.toHTMLline(line))
        return '<br>\n'.join(html)
예제 #22
0
파일: parser.py 프로젝트: CGRU/cgru
    def toHTML(self, i_data):
        """ Convert data to HTML.
            Designed for GUIs for escape sequences, errors highlighting.
        :param i_data: input data
        :return: converted data
        """
        lines = cgruutils.toStr(i_data).replace('\r', '').split('\n')
        html = []

        for line in lines:
            html.append(self.toHTMLline(line))
        return '<br>\n'.join(html)
예제 #23
0
파일: mediainfo.py 프로젝트: CGRU/cgru
def processExif(i_file):
    out = dict()

    out['infile'] = i_file

    if not os.path.isfile(out['infile']):
        out['error'] = 'Input file does not exist.'
        return out

    params = {}
    params['BitDepth'] = 'bitdepth'
    params['ColorType'] = 'colortype'
    params['Compression'] = 'compression'
    params['ImageWidth'] = 'width'
    params['ImageHeight'] = 'height'
    params['Artist'] = 'artist'
    params['Comment'] = 'comments'

    cmd_args = ['exiftool','-S','-fast2']
    for key in params:
        cmd_args.append('-'+key)
    cmd_args.append(i_file)

    try:
        data = subprocess.check_output(cmd_args)
    except:
        out['error'] = 'Failed to execute process.'
        return out

    data = cgruutils.toStr(data)
    data = data.splitlines()
    exif = dict()

    for line in data:
        if len(line) == 0: continue
        fields = line.split(': ', 1)
        if len(fields) != 2: continue
        key = fields[0]
        val = fields[1]
        if len(val) == 0: continue
        if not key in params: continue

        if val.isdigit(): val = int(val)
        else:
            try: val = float(val)
            except: pass

        exif[params[key]] = val


    # return this object:
    out['mediainfo'] = {'exif':exif}
    return out
예제 #24
0
    def parse(self, data, mode):
        """Missing DocString

		:param data:
		:param mode:
		:return:
		"""

        data = cgruutils.toStr(data)
        mode = cgruutils.toStr(mode)

        self.activity = ''
        self.warning = False
        self.error = False
        self.badresult = False
        result = None
        if data.find(str_warning) != -1:
            self.warning = True
        if data.find(str_error) != -1:
            self.error = True
        if data.find(str_badresult) != -1:
            self.badresult = True
        if data.find(str_finishedsuccess) != -1:
            self.finishedsuccess = True

        lines = data.split('\n')
        for line in lines:
            if line.find('@IMAGE@') != -1:
                line = line[7:]
                self.appendFile(line.strip())

        try:
            result = self.do(data, mode)
        except:  # TODO: too broad exception clause
            print('Error parsing output:')
            #print(str(sys.exc_info()[1]))
            traceback.print_exc(file=sys.stdout)
        return result, self.percent, self.frame, self.percentframe, \
            self.warning, self.error, self.badresult, self.finishedsuccess, \
            self.activity
예제 #25
0
	def __init__(self, taskInfo, i_verbose):
		self.taskInfo = taskInfo
		self.verbose = i_verbose

		self.pm = cgrupathmap.PathMap()

		self.str_capacity = str_capacity
		self.str_hosts = str_hosts
		self.str_hostsprefix = str_hostsprefix
		self.str_hostseparator = str_hostseparator

		# Transfer command and working folder:
		command = taskInfo['command']
		command = self.pm.toClient(command)

		# Apply capacity:
		if self.taskInfo['capacity'] > 0:
			command = self.applyCmdCapacity(command)

		# Apply hosts (multihosts tasks):
		if len(self.taskInfo['hosts']):
			command = self.applyCmdHosts(command)
		taskInfo['command'] = command
		taskInfo['wdir'] = self.pm.toClient(taskInfo['wdir'])

		for i in range(0, len(self.taskInfo['files'])):
			self.taskInfo['files'][i] = \
				self.pm.toClient(self.taskInfo['files'][i])

		# When GUI receives task exec to show files,
		# server sends exec with parsed files.
		for i in range(0, len(self.taskInfo['parsed_files'])):
			self.taskInfo['parsed_files'][i] = \
				self.pm.toClient(self.taskInfo['parsed_files'][i])

		# Initialize parser:
		self.parser = None
		parser = cgruutils.toStr(taskInfo['parser'])
		if len(taskInfo['parser']):
			try:
				mod = __import__('parsers', globals(), locals(), [parser])
				cmd = 'mod.%s.%s()' % (parser, parser)
				self.parser = eval(cmd)
				self.parser.setTaskInfo(taskInfo)
			except:  # TODO: too broad exception clause
				self.parser = None
				print('ERROR: Failed to import parser "%s"' % parser)
				traceback.print_exc(file=sys.stdout)

		if self.verbose:
			print(taskInfo)
예제 #26
0
파일: service.py 프로젝트: yazici/cgru
    def __init__(self, taskInfo, i_verbose):
        self.taskInfo = taskInfo
        self.verbose = i_verbose

        self.pm = cgrupathmap.PathMap()

        self.str_capacity = str_capacity
        self.str_hosts = str_hosts
        self.str_hostsprefix = str_hostsprefix
        self.str_hostseparator = str_hostseparator

        # Transfer command and working folder:
        command = taskInfo['command']
        command = self.pm.toClient(command)

        # Apply capacity:
        if self.taskInfo['capacity'] > 0:
            command = self.applyCmdCapacity(command)

        # Apply hosts (multihosts tasks):
        if len(self.taskInfo['hosts']):
            command = self.applyCmdHosts(command)
        taskInfo['command'] = command
        taskInfo['wdir'] = self.pm.toClient(taskInfo['wdir'])

        for i in range(0, len(self.taskInfo['files'])):
            self.taskInfo['files'][i] = \
             self.pm.toClient(self.taskInfo['files'][i])

        # When GUI receives task exec to show files,
        # server sends exec with parsed files.
        for i in range(0, len(self.taskInfo['parsed_files'])):
            self.taskInfo['parsed_files'][i] = \
             self.pm.toClient(self.taskInfo['parsed_files'][i])

        # Initialize parser:
        self.parser = None
        parser = cgruutils.toStr(taskInfo['parser'])
        if len(taskInfo['parser']):
            try:
                mod = __import__('parsers', globals(), locals(), [parser])
                cmd = 'mod.%s.%s()' % (parser, parser)
                self.parser = eval(cmd)
                self.parser.setTaskInfo(taskInfo)
            except:  # TODO: too broad exception clause
                self.parser = None
                print('ERROR: Failed to import parser "%s"' % parser)
                traceback.print_exc(file=sys.stdout)

        if self.verbose:
            print(taskInfo)
예제 #27
0
    def processResources(self, i_args):
        if not 'resources' in i_args:
            return

        resources = None
        try:
            resources = json.loads(cgruutils.toStr(i_args['resources']))
        except:
            print('Bad input resources json:')
            traceback.print_exc(file=sys.stdout)
            resources = None

        if 'host_resources' in resources:
            self.host_resources = resources['host_resources']

        if self.host_resources is None:
            return

        # Peak Memory
        if 'mem_total_mb' and 'mem_free_mb' in self.host_resources:
            mem_used_mb = self.host_resources[
                'mem_total_mb'] - self.host_resources['mem_free_mb']
            if self.res_mem_peak_mb is None or self.res_mem_peak_mb < mem_used_mb:
                self.res_mem_peak_mb = mem_used_mb

        # Average CPU
        cpubusy = None
        for cpu in [
                'cpu_user', 'cpu_nice', 'cpu_system', 'cpu_iowait', 'cpu_irq',
                'cpu_softirq'
        ]:
            if cpu in self.host_resources:
                if cpubusy is None:
                    cpubusy = self.host_resources[cpu]
                else:
                    cpubusy += self.host_resources[cpu]
        if cpubusy is not None:
            self.res_cpu_cur.append(cpubusy)
            self.res_cpu_agv = 0
            for cpu in self.res_cpu_cur:
                self.res_cpu_agv += cpu
            self.res_cpu_agv = int(
                round(self.res_cpu_agv / float(len(self.res_cpu_cur))))
예제 #28
0
파일: events.py 프로젝트: mikedatsik/cgru
    def __init__(self, task_info, i_verbose):
        service.service.__init__(self, task_info, i_verbose)
        data = self.taskInfo['command']
        self.taskInfo['command'] = ''

        # print('Event data:\n%s' % data)

        try:
            if not isinstance(data, str):
                data = str(data, 'utf-8')
            objects = json.loads(data)
        except:  # TODO: Too broad exception clause
            error = str(sys.exc_info()[1])
            print(error)
            print('Event data:\n%s' % data)
            objects = None

        if objects is None:
            return

        print('Event object:')
        print(json.dumps(objects, sort_keys=True, indent=4))

        # Check received events:
        if 'events' not in objects:
            print('ERROR: Received data does not contain events.')
            print('Event data:\n%s' % data)
            return
        if not isinstance(objects['events'], list):
            print('ERROR: Received events is not a list.')
            print('Event data:\n%s' % data)
            return
        if len(objects['events']) == 0:
            print('ERROR: Received events list is empty.')
            print('Event data:\n%s' % data)
            return

        # Get and combine custom data objects:
        custom_obj = dict()
        for key in objects:
            self.combineCustomObj(custom_obj, objects[key])

        print('Combined custom data:')
        print(json.dumps(custom_obj, sort_keys=True, indent=4))

        if len(custom_obj) == 0:
            # print('No configured data found.')
            return

        if 'events' not in custom_obj:
            # print('No configured events found.')
            return

        email_events = []

        # Iterate all interested events:
        for event in custom_obj['events']:

            if event not in objects['events']:
                # print('Skipping not received event "%s"' % event)
                continue

            event_obj = custom_obj['events'][event]

            # Event should be a dictionary:
            if not isinstance(event_obj, dict):
                print('ERROR: Configured event["%s"] is not an object.' %
                      event)
                print('Event data:\n%s' % data)
                return

            if 'methods' not in event_obj:
                print('ERROR: Configured event["%s"] does not have methods.' %
                      event)
                print('Event data:\n%s' % data)
                continue

            methods = event_obj['methods']
            if not isinstance(methods, list):
                print(
                    'ERROR: Configured event["%s"] methods is not an array.' %
                    event)
                print('Event data:\n%s' % data)
                continue

            if 'email' in methods and 'emails' in custom_obj and len(
                    custom_obj['emails']):
                email_events.append(event)

            # Essentially for debugging
            if 'notify-send' in methods:
                self.taskInfo[
                    'command'] = "notify-send Afanasy 'Job " + task_info[
                        'job_name'].replace("'", "'\\''") + ": " + event + "'"

        if len(email_events):
            cmd = cgruconfig.VARS['email_send_cmd']
            cmd += ' -V'  # Verbose mode
            cmd += ' -f "noreply@%s"' % cgruconfig.VARS[
                'email_sender_address_host']
            for addr in custom_obj['emails']:
                cmd += ' -t "%s"' % addr
            cmd += ' -s "%s"' % (','.join(email_events))
            cmd += ' "<p>Events: <b>%s</b></p>"' % (','.join(email_events))
            if 'render' in objects:
                cmd += ' "<p>Render Name: <b>%s</b>' % objects['render']['name']
                if 'host_resources' in objects:
                    hres = objects['host_resources']
                    cmd += '<ul>'
                    cmd += '<li>CPU: %(cpu_mhz)dMHz x%(cpu_num)d / idle = %(cpu_idle)d%%</li>' % hres
                    cmd += '<li>MEM: %dGB / free = %dGB</li>' % (
                        hres['mem_total_mb'] / 1024,
                        hres['mem_free_mb'] / 1024)
                    cmd += '<li>SWP: %dGB / free = %dGB</li>' % (
                        hres['swap_total_mb'] / 1024,
                        (hres['swap_total_mb'] - hres['swap_used_mb']) / 1024)
                    cmd += '<li>HDD: %(hdd_total_gb)dGB / free = %(hdd_free_gb)dGB / busy = %(hdd_busy)d%%</li>' % hres
                    cmd += '</ul>'
                cmd += '</p>"'
            cmd += ' "<p>Job Name: <b>%s</b></p>"' % cgruutils.toStr(
                task_info['job_name'])
            cmd += ' "<p>User Name: <b>%s</b></p>"' % cgruutils.toStr(
                task_info['user_name'])
            print(cmd)
            self.taskInfo['command'] = cmd
예제 #29
0
    errorExit('Unsupported arvhive type "%s"' % Ext)

if CmdPre:
    print(CmdPre)
    os.system(CmdPre)

print(' '.join(CmdList))

Process = subprocess.Popen(CmdList, shell=False, stdout=subprocess.PIPE)

while True:
    data = Process.stdout.readline()
    if data is None: break
    if len(data) < 1: break

    sys.stdout.write(cgruutils.toStr(data))
    sys.stdout.flush()

    if Ext == 'zip':
        data = re.findall('\S+', data)
        if len(data) != 3: continue
        if data[-1] != 'files': continue
        SizeTotal = int(data[0])
        FilesTotal = int(data[1])
    elif Ext == 'rar':
        data = re.findall('\S+', data)
        if len(data) != 5: continue
        try:
            SizeTotal += int(data[1])
            FilesTotal += 1
        except:
예제 #30
0
	def translatePath( self, path, toserver, Verbose):
		newpath = cgruutils.toStr( path)
		if len(newpath) < 1: return newpath
		if not self.initialized: return newpath
		position = 0
		while newpath[position] in PathSeparators:
			position += 1
			if position >= len(newpath): return newpath
		maxcycles = len(newpath)
		cycle = 0
		while position != -1:
			path_search = newpath[position:]
#            print('position # %d/%d : "%s"' % (position, len(newpath), path_search))
			for i in range( 0, len( self.PathServer)):
				if toserver:
					path_from = self.PathClient[i]
					path_to   = self.PathServer[i]
				else:
					path_from = self.PathServer[i]
					path_to   = self.PathClient[i]

				pathfounded = False

				if 'windows' in cgruconfig.VARS['platform'] and toserver:
					path_search = path_search.lower()
					path_from = path_from.replace('/','\\').lower()

				if path_search.find( path_from) == 0:
					pathfounded = True
				else:
					path_from = path_from.replace('\\','/')
					if path_search.find( path_from) == 0:
						pathfounded = True

				if pathfounded:
					part1 = newpath[:position]
					part2 = newpath[position+len(path_from):]
					if not self.UnixSeparators:
						if 'windows' in cgruconfig.VARS['platform'] and not toserver:
							path_to = path_to.replace('/','\\')
					part2 = replaceSeparators( part2, path_from, path_to)
					newpath = part1 + path_to + part2
					position = len(part1 + path_to)
					newpath = part1 + path_to + part2
					if Verbose:
						print('Pathes mapped:')
						print(path)
						print(newpath)
					break

			old_position = position
			position = findNextPosition( position, newpath)
			if position != -1 and position <= old_position:
				print('Path translation error: Eldless cycle, position = %d.' % position)
				break
			cycle += 1
			if cycle > maxcycles:
				print('Path translation error: Cycle > maxcycles (%d>%d).' % (cycle, maxcycles))
				break

		return newpath
예제 #31
0
파일: nvidia_smi.py 프로젝트: stormic/cgru
    def readProcessOuput(self):
        """ Read process stdout
        """
        self.data = None
        obj = dict()

        # Communicate process
        try:
            self.data, err = self.process.communicate()
            if err and len(err):
                print(err)
        except:
            self.setError('Can`t communicate process.', traceback.format_exc())
            return

        # Validate and prepare output data
        self.data = cgruutils.toStr(self.data)
        self.data = self.data.splitlines()
        self.line = 0
        found = False
        while self.line < len(self.data):
            if self.data[self.line].count('<nvidia_smi_log>'):
                found = True
                self.line += 1
                break
            self.line += 1
        if not found:
            self.setError('Ouput does not contain <nvidia_smi_log>.',
                          '\n'.join(self.data))
            return

        # Recursive function to parse XML data and construct dict object:
        self.getLineObj(obj)
        #print(json.dumps(obj, sort_keys=True, indent=4))

        # Process dict object
        mem_total = 0
        mem_used = 0
        mem_free = 0
        tpr_val = 0
        tpr_max = 111

        label = ''
        tip = ''
        for gpu in obj['gpu']:
            if len(label): label += '\n'
            if len(tip): tip += '\n'

            label += gpu['product_name']
            tip += gpu['product_name']

            # Memory (used and total):
            mem = gpu['fb_memory_usage']
            total = int(mem['total'].split(' ')[0])
            used = int(mem['used'].split(' ')[0])
            free = int(mem['free'].split(' ')[0])
            label += ' %.0fG (%dM Used / %dM Free)' % (float(total) / 1000.0,
                                                       used, free)
            tip += ' Memory: Total %dM, Used %dM, Free %dM' % (total, used,
                                                               free)
            mem_total += total
            mem_used += used
            mem_free += free

            # Temperature (current and max):
            tpr = gpu['temperature']
            gpu_tpr = int(tpr['gpu_temp'].split(' ')[0])
            gpu_tpr_max = int(tpr['gpu_temp_slow_threshold'].split(' ')[0])
            label += ' %dC(%dC Max)' % (gpu_tpr, gpu_tpr_max)
            tip += '; Temperature: GPU %dC, Max %dC' % (gpu_tpr, gpu_tpr_max)
            if tpr_val < gpu_tpr: tpr_val = gpu_tpr
            if tpr_max > gpu_tpr_max: tpr_max = gpu_tpr_max

            # Utilization:
            util = gpu['utilization']
            util_gpu = int(util['gpu_util'].split(' ')[0])
            util_mem = int(util['memory_util'].split(' ')[0])
            util_enc = int(util['encoder_util'].split(' ')[0])
            util_dec = int(util['decoder_util'].split(' ')[0])
            if util_gpu: label += ' G%d%%' % util_gpu
            if util_mem: label += ' M%d%%' % util_mem
            if util_enc: label += ' E%d%%' % util_enc
            if util_dec: label += ' D%d%%' % util_dec
            tip += '; Utilization: GPU:%d%% MEM:%d%% Encoder:%d%% Decoder:%d%%' % (
                util_gpu, util_mem, util_enc, util_dec)

            # Collect processes and progs (processes with the same name)
            if not 'process_info' in gpu['processes']: continue
            self.labelg = 255
            processes = []
            progs = {}

            for prc in gpu['processes']['process_info']:
                iterate = False
                try:
                    if isinstance(prc, dict):
                        name = prc['process_name']
                        mem = int(prc['used_memory'].split(' ')[0])
                        iterate = True
                    else:
                        name = gpu['processes']['process_info']['process_name']
                        mem = int(gpu['processes']['process_info']
                                  ['used_memory'].split(' ')[0])
                except:
                    print(str(gpu['processes']['process_info']))
                    print(traceback.format_exc())
                    name = str(prc)
                    mem = 0
                    self.labelr = 255
                    self.labelg = 0
                    self.labelb = 0

                # append processes:
                pc = dict()
                pc['name'] = name
                pc['mem'] = mem
                processes.append(pc)
                # Cut command arguments:
                name = name.strip().split(' ')[0]
                # Use base name for progs:
                name = name.split('/')[-1].split('\\')[-1]
                if name in progs:
                    progs[name] += mem
                else:
                    progs[name] = mem

                if not iterate:
                    break

            # constuct label string (sorted by mem):
            label += '\n'
            for name, mem in sorted(progs.items(),
                                    key=lambda kv: kv[1],
                                    reverse=True):
                label += ' ' + name
                label += ':%dM' % mem

            # constuct tip strings (sorted by mem):
            processes.sort(key=lambda k: k['mem'], reverse=True)
            for prc in processes:
                tip += '\n' + prc['name']
                tip += ': %d MiB' % prc['mem']

        self.label = 'v' + obj['driver_version'] + ' ' + label
        self.tooltip = 'NVIDIA Driver Verion: ' + obj[
            'driver_version'] + '\n' + tip

        if mem_used:
            self.labelr = 255

        if mem_total:
            self.valuemax = mem_total
            self.value = mem_used
            clr = getClr(mem_used, mem_total, self.mem_clr_min,
                         self.mem_clr_max)
            r = 2.0 * clr
            if r > 1.0: r = 1.0
            g = 2.0 * clr
            if g < 1.0: g = 1.0
            g = 2.0 - g
            self.graphr = int(255 * r)
            self.graphg = int(255 * g)

        if tpr_val:
            clr = getClr(tpr_val, tpr_max, self.tpr_clr_min, self.tpr_clr_max)
            self.bgcolorb = int(255 * clr)
예제 #32
0
    def __init__(self, taskInfo, i_verbose):
        self.taskInfo = taskInfo
        self.verbose = i_verbose
        self.log = None

        self.pm = cgrupathmap.PathMap()

        self.str_capacity = str_capacity
        self.str_hosts = str_hosts
        self.str_hostsprefix = str_hostsprefix
        self.str_hostseparator = str_hostseparator

        # Transfer command and working folder:
        command = taskInfo['command']
        command = self.pm.toClient(command)

        # Apply capacity:
        if self.taskInfo['capacity'] > 0:
            command = self.applyCmdCapacity(command)

        # Apply hosts (multihosts tasks):
        if len(self.taskInfo['hosts']):
            command = self.applyCmdHosts(command)
        taskInfo['command'] = command
        taskInfo['wdir'] = self.pm.toClient(taskInfo['wdir'])

        for i in range(0, len(self.taskInfo['files'])):
            self.taskInfo['files'][i] = \
             self.pm.toClient(self.taskInfo['files'][i])

        # Check files:
        if self.isSkippingExistingFiles():
            allFilesExist = True
            for i in range(0, len(self.taskInfo['files'])):
                afile = self.taskInfo['files'][i]
                afile = os.path.join(taskInfo['wdir'], afile)
                if not os.path.isfile(afile):
                    allFilesExist = False
                    break
                # Check files size:
                file_size_min = self.taskInfo['file_size_min']
                file_size_max = self.taskInfo['file_size_max']
                if file_size_min > 0 or file_size_max > 0:
                    size = os.path.getsize(afile)
                    if file_size_min > 0 and size < file_size_min:
                        allFilesExist = False
                        break
                    if file_size_max > 0 and size > file_size_max:
                        allFilesExist = False
                        break

            if allFilesExist:
                self.log = 'Task file(s) exits.'
                self.taskInfo['command'] = ''

        # When GUI receives task exec to show files,
        # server sends exec with parsed files.
        for i in range(0, len(self.taskInfo['parsed_files'])):
            self.taskInfo['parsed_files'][i] = \
             self.pm.toClient(self.taskInfo['parsed_files'][i])

        # Initialize parser:
        self.parser = None
        parser = cgruutils.toStr(taskInfo['parser'])
        if len(taskInfo['parser']):
            try:
                mod = __import__('parsers', globals(), locals(), [parser])
                cmd = 'mod.%s.%s()' % (parser, parser)
                self.parser = eval(cmd)
                self.parser.setTaskInfo(taskInfo)
            except:  # TODO: too broad exception clause
                self.parser = None
                print('ERROR: Failed to import parser "%s"' % parser)
                traceback.print_exc(file=sys.stdout)

        if self.verbose:
            print(taskInfo)
예제 #33
0
    errorExit('Unsupported arvhive type "%s"' % Ext)

if CmdPre:
    print(CmdPre)
    os.system(CmdPre)

print(' '.join(CmdList))

Process = subprocess.Popen(CmdList, shell=False, stdout=subprocess.PIPE)

while True:
    data = Process.stdout.readline()
    if data is None: break
    if len(data) < 1: break

    data = cgruutils.toStr(data)
    sys.stdout.write(data)
    sys.stdout.flush()

    if Ext == 'zip':
        data = re.findall('\S+', data)
        if len(data) != 3: continue
        if data[-1] != 'files': continue
        SizeTotal = int(data[0])
        FilesTotal = int(data[1])
    elif Ext == 'rar':
        data = re.findall('\S+', data)
        if len(data) != 5: continue
        try:
            SizeTotal += int(data[1])
            FilesTotal += 1
예제 #34
0
파일: arch.py 프로젝트: stormic/cgru
print(' '.join(Cmd))

if Options.debug:
	sys.exit(0)

if os.path.isfile(Options.output):
	print('Deleting exising archive:\n' + Options.output)
	os.remove(Options.output)

Process = subprocess.Popen( Cmd, shell=False, stdout=subprocess.PIPE)

ThumbTime = 0

while True:
	data = cgruutils.toStr(Process.stdout.readline())
	if data is None: break
	if len(data) < 1: break

	sys.stdout.write(data)

	if data.find(Key) != -1 and FilesTotal:
		if data not in Files:
			print('PROGRESS: %d%%' % int( 100.0 * len(Files) / FilesTotal))
			Files.append( data)

			curtime = time.time()
			if Options.thumbsec is not None and curtime - ThumbTime > Options.thumbsec:
				data = data[data.find(Key):]
				data = data[len(Key):]
				data = data.strip(' \n\r')
예제 #35
0
    def __init__(self, taskInfo, i_verbose):
        self.taskInfo = taskInfo
        self.verbose = i_verbose
        self.log = None

        self.numeric = afcommon.checkBlockFlag(taskInfo['block_flags'],
                                               'numeric')
        if self.verbose: print(taskInfo)

        self.pm = cgrupathmap.PathMap()

        self.str_capacity = str_capacity
        self.str_hosts = str_hosts
        self.str_hostsprefix = str_hostsprefix
        self.str_hostseparator = str_hostseparator

        # Transfer working folder:
        taskInfo['wdir'] = self.pm.toClient(taskInfo['wdir'])

        # Process command:
        command = self.processCommandPattern()

        # Transfer command:
        command = self.pm.toClient(command)

        # Apply capacity:
        if self.taskInfo['capacity'] > 0:
            command = self.applyCmdCapacity(command)

        # Apply hosts (multihosts tasks):
        if len(self.taskInfo['hosts']):
            command = self.applyCmdHosts(command)

        taskInfo['command'] = command
        if self.verbose:
            print('Processed task command:\n' + command)

        # Process files:
        taskInfo['files'] = self.processFilesPattern()
        if self.verbose:
            print('Processed task files:')
            for afile in taskInfo['files']:
                print(afile)

        # Transfer paths in files:
        for i in range(0, len(self.taskInfo['files'])):
            self.taskInfo['files'][i] = \
                self.pm.toClient(self.taskInfo['files'][i])

        # Check files:
        if self.isSkippingExistingFiles() and len(self.taskInfo['files']):
            self.checkExistingFiles()

        # Transfer paths in environment:
        for name in self.taskInfo['environment']:
            self.taskInfo['environment'][name] = self.pm.toClient(
                self.taskInfo['environment'][name])

        # When GUI receives task exec to show files,
        # server sends exec with parsed files.
        for i in range(0, len(self.taskInfo['parsed_files'])):
            self.taskInfo['parsed_files'][i] = \
                self.pm.toClient(self.taskInfo['parsed_files'][i])

        # Initialize parser:
        self.parser = None
        parser = cgruutils.toStr(taskInfo['parser'])
        if len(taskInfo['parser']):
            try:
                mod = __import__('parsers', globals(), locals(), [parser])
                cmd = 'mod.%s.%s()' % (parser, parser)
                self.parser = eval(cmd)
                self.parser.setTaskInfo(taskInfo)
            except:  # TODO: too broad exception clause
                self.parser = None
                print('ERROR: Failed to import parser "%s"' % parser)
                traceback.print_exc(file=sys.stdout)

        if self.verbose and self.log and len(self.log):
            print(self.log)
예제 #36
0
    def update(self):
        if self.process is not None:
            out, err = self.process.communicate()
            data = cgruutils.toStr(out)
            data = data.replace(',','.')
            data = data.splitlines()

            fields_pos = -1
            for i in range(1, len(data)):
                i = len(data) - i
                if data[i][:6] == 'Device':
                    fields_pos = i
                    break
            if fields_pos == -1:
                print('ERROR: iostat: Can not find "Device" in output lines:')
                print(data)
                return
            if fields_pos >= (len(data) - 1):
                print('ERROR: iostat: Can not find any devices.')
                print(data)
                return
            fieldsline = data[fields_pos]
            fields = fieldsline.split()

            matcheddevices = ''
            rMBs = 0.0
            wMBs = 0.0
            util = 0.0
            avgrqsz = 0.0
            avgqusz = 0.0
            awaitsz = 0.0
            svctm = 0.0

            for i in range(fields_pos + 1, len(data) - 1):
                values = data[i].split()
                if len(values) < 1:
                    continue
                device = values[0]
                match = self.regexp.match(device)
                if match is None:
                    continue
                if match.group(0) != device:
                    continue
                matcheddevices += ' ' + device
                try:
                    f_rMBs = float(values[COL_rMBs])
                    f_wMBs = float(values[COL_wMBs])
                    f_util = float(values[COL_util])
                    f_avgrqsz = float(values[COL_avgrqsz])
                    f_avgqusz = float(values[COL_avgqusz])
                    f_awaitsz = float(values[COL_awaitsz])
                    f_svctm = float(values[COL_svctm])
                except:  # TODO: too broad exception clause
                    print('ERROR: iostat: Invalid columns values formatting.')
                    traceback.print_exc(file=sys.stdout)
                    continue
                rMBs += f_rMBs
                wMBs += f_wMBs
                if f_util > util:
                    util = f_util
                if f_avgrqsz > avgrqsz:
                    avgrqsz = f_avgrqsz
                if f_avgqusz > avgqusz:
                    avgqusz = f_avgqusz
                if f_awaitsz > awaitsz:
                    awaitsz = f_awaitsz
                if f_svctm > svctm:
                    svctm = f_svctm

            if matcheddevices != '':
                self.label = matcheddevices + ':'
                self.label += ' %s=%.2f' % (fields[COL_rMBs], rMBs)
                self.label += ' %s=%.2f' % (fields[COL_wMBs], wMBs)
                self.label += ' %s=%.2f' % (fields[COL_util], util)
                self.label += '\n'
                self.label += ' %s=%.2f' % (fields[COL_avgrqsz], avgrqsz)
                self.label += ' %s=%.2f' % (fields[COL_avgqusz], avgqusz)
                self.label += ' %s=%.2f' % (fields[COL_awaitsz], awaitsz)
                self.label += ' %s=%.2f' % (fields[COL_svctm], svctm)
                self.value = int(util)
                if self.value > self.valuemax:
                    self.value = self.valuemax
                if self.value < 0:
                    self.value = 0
                self.graphr = int(5 * self.value / 2)
                self.graphg = 255 - int(5 * self.value / 2)
                self.graphb = 10
                self.bgcolorb = self.value
            else:
                self.label = 'No such devices found: "%s"' % self.device

        self.process = subprocess.Popen(
            ['iostat', '-x', '-d', '-m', '4', '2'],
            stdout=subprocess.PIPE
        )
예제 #37
0
	def translatePath(self, path, toserver, Verbose):
		"""Missing DocString

		:param path:
		:param toserver:
		:param Verbose:
		:return:
		"""
		newpath = cgruutils.toStr(path)
		if len(newpath) < 1:
			return newpath

		if not self.initialized:
			return newpath

		position = 0

		while newpath[position] in PathSeparators:
			position += 1
			if position >= len(newpath):
				return newpath

		maxcycles = len(newpath)
		cycle = 0
		while position != -1:
			path_search = newpath[position:]
			# print('position # %d/%d : "%s"' % (position, len(newpath), path_search))
			for i in range(0, len(self.PathServer)):
				if toserver:
					path_from = self.PathClient[i]
					path_to = self.PathServer[i]
				else:
					path_from = self.PathServer[i]
					path_to = self.PathClient[i]

				path_found = False

				if 'windows' in cgruconfig.VARS['platform'] and toserver:
					path_search = path_search.lower()
					path_from = path_from.replace('/', '\\').lower()

				if path_search.find(path_from) == 0:
					path_found = True
				else:
					path_from = path_from.replace('\\', '/')
					if path_search.find(path_from) == 0:
						path_found = True

				if path_found:
					part1 = newpath[:position]
					part2 = newpath[position + len(path_from):]
					if not self.UnixSeparators:
						if 'windows' in cgruconfig.VARS['platform'] \
								and not toserver:
							path_to = path_to.replace('/', '\\')
					part2 = replaceSeparators(part2, path_from, path_to)
					position = len(part1 + path_to)
					newpath = part1 + path_to + part2
					if Verbose:
						print('Paths mapped:')
						print(path)
						print(newpath)
					break

			old_position = position
			position = findNextPosition(position, newpath)
			if position != -1 and position <= old_position:
				print('Path translation error: Eldless cycle, '
					  'position = %d.' % position)
				break

			cycle += 1

			if cycle > maxcycles:
				print(
					'Path translation error: Cycle > maxcycles (%d>%d).' %
					(cycle, maxcycles)
				)
				break

		return newpath
예제 #38
0
파일: service.py 프로젝트: fsimerey/cgru
    def __init__(self, taskInfo, i_verbose):
        self.taskInfo = taskInfo
        self.verbose = i_verbose
        self.log = None

        self.pm = cgrupathmap.PathMap()

        self.str_capacity = str_capacity
        self.str_hosts = str_hosts
        self.str_hostsprefix = str_hostsprefix
        self.str_hostseparator = str_hostseparator

        # Transfer command and working folder:
        command = taskInfo['command']
        command = self.pm.toClient(command)

        # Apply capacity:
        if self.taskInfo['capacity'] > 0:
            command = self.applyCmdCapacity(command)

        # Apply hosts (multihosts tasks):
        if len(self.taskInfo['hosts']):
            command = self.applyCmdHosts(command)
        taskInfo['command'] = command
        taskInfo['wdir'] = self.pm.toClient(taskInfo['wdir'])

        for i in range(0, len(self.taskInfo['files'])):
            self.taskInfo['files'][i] = \
                self.pm.toClient(self.taskInfo['files'][i])

        # Check files:
        if self.isSkippingExistingFiles():
            allFilesExist = True
            for i in range(0, len(self.taskInfo['files'])):
                afile = self.taskInfo['files'][i]
                afile = os.path.join( taskInfo['wdir'], afile)
                if not os.path.isfile( afile):
                    allFilesExist = False
                    break
                # Check files size:
                file_size_min = self.taskInfo['file_size_min']
                file_size_max = self.taskInfo['file_size_max']
                if file_size_min > 0 or file_size_max > 0:
                    size = os.path.getsize(afile)
                    if file_size_min > 0 and size < file_size_min:
                        allFilesExist = False
                        break
                    if file_size_max > 0 and size > file_size_max:
                        allFilesExist = False
                        break

            if allFilesExist:
                self.log = 'Task file(s) exits.'
                self.taskInfo['command'] = ''

        # When GUI receives task exec to show files,
        # server sends exec with parsed files.
        for i in range(0, len(self.taskInfo['parsed_files'])):
            self.taskInfo['parsed_files'][i] = \
                self.pm.toClient(self.taskInfo['parsed_files'][i])

        # Initialize parser:
        self.parser = None
        parser = cgruutils.toStr(taskInfo['parser'])
        if len(taskInfo['parser']):
            try:
                mod = __import__('parsers', globals(), locals(), [parser])
                cmd = 'mod.%s.%s()' % (parser, parser)
                self.parser = eval(cmd)
                self.parser.setTaskInfo(taskInfo)
            except:  # TODO: too broad exception clause
                self.parser = None
                print('ERROR: Failed to import parser "%s"' % parser)
                traceback.print_exc(file=sys.stdout)

        if self.verbose:
            print(taskInfo)
예제 #39
0
파일: afnetwork.py 프로젝트: nDeedy/cgru
def sendServer(i_data, i_verbose=False):
    """Missing DocString

    :param i_verbose:
    :return:
    """
    size = len(i_data)
    header = genHeader(size)
    i_data = header + bytearray(i_data, 'utf-8')
    datalen = len(i_data)
    # return True, None

    host = cgruconfig.VARS['af_servername']
    port = cgruconfig.VARS['af_serverport']

    s = None
    err_msg = ''
    reslist = []

    try:
        reslist = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
                                     socket.SOCK_STREAM)
    except:  # TODO: Too broad exception clause
        print('Can`t solve "%s":' % host + str(sys.exc_info()[1]))

    for res in reslist:
        af, socktype, proto, canonname, sa = res
        if i_verbose:
            print('Trying to connect to "%s"' % str(sa[0]))
        try:
            s = socket.socket(af, socktype, proto)
        except:  # TODO: Too broad exception clause
            if err_msg != '':
                err_msg += '\n'
            err_msg += str(sa[0]) + ' : ' + str(sys.exc_info()[1])
            s = None
            continue
        try:
            s.connect(sa)
        except:  # TODO: Too broad exception clause
            if err_msg != '':
                err_msg += '\n'
            err_msg += str(sa[0]) + ' : ' + str(sys.exc_info()[1])
            s.close()
            s = None
            continue
        break

    if s is None:
        print('Could not open socket.')
        print(err_msg)
        return False, None

    if i_verbose:
        print('afnetwork.sendServer: send %d bytes' % datalen)

    # s.sendall( i_data) #<<< !!! May not work !!!!

    total_send = 0
    while total_send < len(i_data):
        sent = s.send(i_data[total_send:])
        if sent == 0:
            disconnectSocket(s)
            print('Error: Unable send data to socket')
            return False, None
        total_send += sent

    data = b''
    msg_len = None
    while True:
        data_buffer = s.recv(4096)

        if not data_buffer:
            break

        data += data_buffer

        if msg_len is None:
            dataStr = cgruutils.toStr(data)
            if dataStr.find('AFANASY') != -1 and dataStr.find('JSON') != -1:
                msg_len = dataStr[:dataStr.find('JSON') + 4]
                msg_len = len(msg_len) + int(msg_len.split(' ')[1])

        if i_verbose:
            print('Received %d of %d bytes.' % (len(data), msg_len))

        if msg_len is not None:
            if len(data) >= msg_len:
                break

    disconnectSocket(s)

    struct = None

    try:
        if not isinstance(data, str):
            data = cgruutils.toStr(data)
        if msg_len is not None:
            data = data[data.find('JSON') + 4:msg_len]
        struct = json.loads(data, strict=False)
    except:  # TODO: Too broad exception clause
        print('afnetwork.py: Received data:')
        print(data)
        print('JSON loads error:')
        print(str(sys.exc_info()[1]))
        struct = None

    return True, struct
예제 #40
0
def sendServer(data, receive=True, verbose=False):
	"""Missing DocString

	:param receive:
	:param verbose:
	:return:
	"""
	size = len(data)
	header = genHeader(size)
	data = header + bytearray(data, 'utf-8')
	datalen = len(data)
	# return True, None

	host = cgruconfig.VARS['af_servername']
	port = cgruconfig.VARS['af_serverport']

	s = None
	err_msg = ''
	reslist = []

	try:
		reslist = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
									 socket.SOCK_STREAM)
	except:  # TODO: Too broad exception clause
		print('Can`t solve "%s":' % host + str(sys.exc_info()[1]))

	for res in reslist:
		af, socktype, proto, canonname, sa = res
		if verbose:
			print('Trying to connect to "%s"' % str(sa[0]))
		try:
			s = socket.socket(af, socktype, proto)
		except:  # TODO: Too broad exception clause
			if err_msg != '':
				err_msg += '\n'
			err_msg += str(sa[0]) + ' : ' + str(sys.exc_info()[1])
			s = None
			continue
		try:
			s.connect(sa)
		except:  # TODO: Too broad exception clause
			if err_msg != '':
				err_msg += '\n'
			err_msg += str(sa[0]) + ' : ' + str(sys.exc_info()[1])
			s.close()
			s = None
			continue
		break

	if s is None:
		print('Could not open socket.')
		print(err_msg)
		return False, None

	if verbose:
		print('afnetwork.sendServer: send %d bytes' % datalen)

	# s.sendall( data) #<<< !!! May not work !!!!

	total_send = 0
	while total_send < len(data):
		sent = s.send(data[total_send:])
		if sent == 0:
			disconnectSocket( s)
			print('Error: Unable send data to socket')
			return False, None
		total_send += sent

	if not receive:
		disconnectSocket( s)
		return True, None


	data = b''
	msg_len = None
	while True:
		buffer = s.recv(4096)

		if not buffer:
			break

		data += buffer

		if msg_len is None:
			dataStr = cgruutils.toStr(data)
			if dataStr.find('AFANASY') != -1 and dataStr.find('JSON') != -1:
				msg_len = dataStr[:dataStr.find('JSON')+4]
				msg_len = len(msg_len) + int(msg_len.split(' ')[1])

		if verbose:
			print('Received %d of %d bytes.' % ( len(data), msg_len))

		if msg_len is not None:
			if len( data) >= msg_len:
				break

	disconnectSocket( s)

	struct = None

	try:
		if not isinstance(data, str):
			data = cgruutils.toStr(data)
		data = data[data.find('JSON')+4:]
		struct = json.loads(data)
	except:  # TODO: Too broad exception clause
		print('afnetwork.py: Received data:')
		print(data)
		print('JSON loads error:')
		print(str(sys.exc_info()[1]))
		struct = None

	return True, struct
예제 #41
0
Parser_Error = False
Parser_BadResult = False
Parser_Warning = False
Parser_Activity = None
Parser_Resources = None
Parser_Report = None

while True:
    stdout.flush()
    data = stdout.readline()
    if data is None:
        break
    if len(data) < 1:
        break

    printMuted(cgruutils.toStr(data))

    parser.parse({'data': data})

    info = 'Parse:'
    info += ' %d%%: %d frame %d%%;' % (parser.percent, parser.frame,
                                       parser.percentframe)

    if len(parser.activity):
        info += ' Activity: %s;' % parser.activity
        Parser_Activity = parser.activity

    if len(parser.resources):
        info += ' Resources: %s;' % parser.resources
        Parser_Resources = parser.resources
예제 #42
0
파일: nvidia_smi.py 프로젝트: CGRU/cgru
    def readProcessOuput(self):
        """ Read process stdout
        """
        self.data = None
        obj = dict()

        # Communicate process
        try:
            self.data, err = self.process.communicate()
            if err and len(err):
                print(err)
        except:
            self.setError('Can`t communicate process.', traceback.format_exc())
            return

        # Validate and prepare output data    
        self.data = cgruutils.toStr(self.data)
        self.data = self.data.splitlines()
        self.line = 0
        found = False
        while self.line < len(self.data):
            if self.data[self.line].count('<nvidia_smi_log>'):
                found = True
                self.line += 1
                break
            self.line += 1
        if not found:
            self.setError('Ouput does not contain <nvidia_smi_log>.', '\n'.join(self.data))
            return

        # Recursive function to parse XML data and construct dict object:
        self.getLineObj(obj)
        #print(json.dumps(obj, sort_keys=True, indent=4))

        # Process dict object
        mem_total = 0
        mem_used  = 0
        mem_free  = 0
        tpr_val = 0
        tpr_max = 111

        label = ''
        tip = ''
        for gpu in obj['gpu']:
            if len(label): label += '\n'
            if len(tip): tip += '\n'

            label += gpu['product_name']
            tip += gpu['product_name']

            # Memory (used and total):
            mem = gpu['fb_memory_usage']
            total = int(mem['total'].split(' ')[0])
            used  = int(mem['used' ].split(' ')[0])
            free  = int(mem['free' ].split(' ')[0])
            label += ' %.0fG (%dM Used / %dM Free)' % (float(total)/1000.0,used,free)
            tip += ' Memory: Total %dM, Used %dM, Free %dM' % (total,used,free)
            mem_total += total
            mem_used  += used
            mem_free  += free

            # Temperature (current and max):
            tpr = gpu['temperature']
            gpu_tpr = int(tpr['gpu_temp'].split(' ')[0])
            gpu_tpr_max = int(tpr['gpu_temp_slow_threshold'].split(' ')[0])
            label += ' %dC(%dC Max)' % (gpu_tpr, gpu_tpr_max)
            tip += '; Temperature: GPU %dC, Max %dC' % (gpu_tpr, gpu_tpr_max)
            if tpr_val < gpu_tpr: tpr_val = gpu_tpr
            if tpr_max > gpu_tpr_max: tpr_max = gpu_tpr_max

            # Utilization:
            util = gpu['utilization']
            util_gpu = int(util['gpu_util'].split(' ')[0])
            util_mem = int(util['memory_util'].split(' ')[0])
            util_enc = int(util['encoder_util'].split(' ')[0])
            util_dec = int(util['decoder_util'].split(' ')[0])
            if util_gpu: label += ' G%d%%' % util_gpu
            if util_mem: label += ' M%d%%' % util_mem
            if util_enc: label += ' E%d%%' % util_enc
            if util_dec: label += ' D%d%%' % util_dec
            tip += '; Utilization: GPU:%d%% MEM:%d%% Encoder:%d%% Decoder:%d%%' % (util_gpu, util_mem, util_enc, util_dec)

            # Collect processes and progs (processes with the same name)
            if not 'process_info' in gpu['processes']: continue
            self.labelg = 255
            processes = []
            progs = {}

            for prc in gpu['processes']['process_info']:
                iterate = False
                try:
                    if isinstance(prc, dict):
                        name = prc['process_name']
                        mem = int(prc['used_memory'].split(' ')[0])
                        iterate = True
                    else:
                        name = gpu['processes']['process_info']['process_name']
                        mem = int(gpu['processes']['process_info']['used_memory'].split(' ')[0])
                except:
                    print(str(gpu['processes']['process_info']))
                    print(traceback.format_exc())
                    name = str(prc)
                    mem = 0
                    self.labelr = 255
                    self.labelg = 0
                    self.labelb = 0

                # append processes:
                pc = dict()
                pc['name'] = name
                pc['mem'] = mem
                processes.append(pc)
                # Cut command arguments:
                name = name.strip().split(' ')[0]
                # Use base name for progs:
                name = name.split('/')[-1].split('\\')[-1]
                if name in progs:
                    progs[name] += mem
                else:
                    progs[name] = mem

                if not iterate:
                    break

            # constuct label string (sorted by mem):
            label += '\n'
            for name, mem in sorted(progs.items(), key=lambda kv: kv[1], reverse=True):
                label += ' ' + name
                label += ':%dM' % mem

            # constuct tip strings (sorted by mem):
            processes.sort(key=lambda k: k['mem'], reverse=True)
            for prc in processes:
                tip += '\n' + prc['name']
                tip += ': %d MiB' % prc['mem']


        self.label = 'v' + obj['driver_version'] + ' ' + label
        self.tooltip = 'NVIDIA Driver Verion: ' + obj['driver_version'] + '\n' + tip

        if mem_used:
            self.labelr = 255

        if mem_total:
            self.valuemax = mem_total
            self.value = mem_used
            clr = getClr(mem_used, mem_total, self.mem_clr_min, self.mem_clr_max)
            r = 2.0 * clr
            if r > 1.0: r = 1.0
            g = 2.0 * clr
            if g < 1.0: g = 1.0
            g = 2.0 - g
            self.graphr = int(255*r)
            self.graphg = int(255*g)

        if tpr_val:
            clr = getClr(tpr_val, tpr_max, self.tpr_clr_min, self.tpr_clr_max)
            self.bgcolorb = int(255*clr)