def __init__(self, process_count=8): """初始化,通过process_count 参数指定最多使用的进程数量""" self.process_count = process_count self.disable_print = os.environ.get('disable_print', '0') self.process_info = {} #self.logger = create_logger('MultiProcessRunner') self.logger = utils.getLogger('RES')
class ProgressLogger(Singleton): _logger = utils.getLogger('job') _width = 60 _error_list = [] _warning_list = [] def setLogName(self, name): self._logger = utils.getLogger(name) def setTotal(self, cnt): self._total = cnt self._count = 0 def __next__(self): # return self._count += 1 def debug(self, s): self.log(s, 'debug') def info(self, s): self.log(s, 'info') def warning(self, s): self.log(s, 'warning') def error(self, s): self.log(s, 'error') def log(self, s, level): if level == 'error': self._error_list.append(s) if level == 'warning': self._warning_list.append(s) if hasattr(self, '_total') and self._total > 0: sys.stdout.write(' ' * (self._width + 9) + '\r') sys.stdout.flush() getattr(self._logger, level)(s) if not (hasattr(self, '_total') and self._total > 0): return progress = self._width * self._count / self._total sys.stdout.write('{0:3}/{1:3}: '.format(self._count, self._total)) sys.stdout.write('#' * int(progress) + '-' * int(self._width - progress) + '\r') # if self._count == self._total: # sys.stdout.write('\n') sys.stdout.flush() def export_error_logs(self): for i in range(len(self._warning_list)): self._logger.warning(self._warning_list[i]) for i in range(len(self._error_list)): self._logger.error(self._error_list[i])
#coding=utf-8 import click import sys import os import os.path from rtool import utils from rtool.pcutils import cache_tool # from tkinter import * # from tkinter.filedialog import askdirectory logger = utils.getLogger('Demo') def run(click_group): from rtool.tasks import task click_group.add_command(task.run_dev, name='task_dev') click_group.add_command(task.run, name='task')
from rtool.tasks.Base import Base from rtool.tasks.BasePostAction import output_to_temp from rtool.tasks.BaseStatus import BaseStatus from rtool import utils from rtool.pcutils.md5sum_of_path import md5_for_str from rtool.pcutils.md5sum_of_path import md5_for_file from rtool.pcutils.md5sum_of_path import md5sum_of_path from rtool.pcutils.md5sum_of_path import md5sum_of_dir from rtool.pcutils.md5sum_of_path import uni_id_for_res RC_ADDRESS = utils.getMainConfig().get("RC_ADDRESS", "") RC_ROOT = utils.getMainConfig().get( "RC_ROOT", "/data/home/user00/service/resource_center") # SECRETS_FILE = "/etc/rsyncd.secrets" # WORKER_USER = "******" logger = utils.getLogger("BaseSync") RSYNC_TEMPLATE = "rsync -avzP $src $dst " RSYNC_TEMPLATE_WITH_EXCLUDE = "rsync -avzP --exclude $exclude $src $dst " MAKE_RC_DIR_TMEPLATE = "ssh $rc_address 'mkdir -p $rc_dir'" DB_CONFIG = { "host": "172.16.153.49", "port": 51000, "user": "******", "password": "******", "db": "asset_vega" } DB_CONFIG = utils.getMainConfig().get("DB_CONFIG", DB_CONFIG) DB_CONN = None
#coding=utf-8 import os import os.path import imp import rtool.utils as mut logger = mut.getLogger('BaseActionDispatch') class BaseActionDispatch(object): """docstring for BaseActionDispatch""" def __init__(self): super(BaseActionDispatch, self).__init__() self.plugindir=[] def set_plugin_dir(self,plugindir): self.plugindir = plugindir def invokeAction(self,module,configs,tp=None,plugindir=[]): dirlist=[] if len(plugindir)>0: dirlist = plugindir else: dirlist = self.plugindir if not len(dirlist)>0: logger.warning(u"插件目录未设置") return file,pathname,dec = imp.find_module(module,dirlist) taskModule = imp.load_module(module,file,pathname,dec) taskModule.run_with_configs(configs,tp) pass
#coding=utf-8 from rtool import utils from rtool.taskplugin.plugin.SampleAction import SampleAction import json logger = utils.getLogger('CustomAction') def run_with_configs(configs, tp=None): logger.debug("Executing CustomAction") apaction = CustomPluginDemo() apaction.go(configs) pass def clean_output(configs): default_output_path = configs["output-dir"] class CustomPluginDemo(SampleAction): """docstring for CustomPluginDemo""" def go(self, config): SampleAction.go(self, config) logger.warning("CustomPluginDemo ") logger.warning(json.dumps(config, indent=2))
def __init__(self): self.logger = utils.getLogger(__name__)
#coding=utf-8 # 最简示例,用于展示action插件的编写方法 import yaml import os import os.path import shutil import json import subprocess import sys sys.path.append(os.path.split(os.path.realpath(__file__))[0]) import MultiProcessRunner import rtool.utils as utils logger = utils.getLogger('SampleAction') def run(): logger.debug("SampleAction") pass def run_with_configs(configs, tp=None): logger.debug("Executing SampleAction") apaction = SampleAction() apaction.go(configs) pass def clean_output(configs): default_output_path = configs["output-root"]
#coding=utf-8 # 使用CCSUIExporter将csd转化为csb import yaml import os import os.path import shutil import json import subprocess import sys import re sys.path.append(os.path.split(os.path.realpath(__file__))[0]) import rtool.taskplugin.plugin.MultiProcessRunner as MultiProcessRunner import rtool.utils as utils logger = utils.getLogger('MakeCSBAction') def run(): logger.debug("MakeCSBAction") pass def run_with_configs(configs, tp=None): logger.debug("Executing MakeCSBAction") apaction = MakeCSBAction() apaction.go(configs) pass def clean_output(configs): default_output_path = configs["output-root"]
from rtool import utils import click import rtool.utils as mut import os import os.path import rtool.pcutils.md5sum_of_path as mput import sys import json import rtool.res.TPCache2 as TPCache2 import rtool.res.TPTask as TPTask from rtool.tasks.task import AbstractTask from rtool.tasks import BasePreAction from rtool.tasks.BaseParseConfigs import BaseParseConfigs from rtool.tasks.BasePostAction import BasePostAction logger = utils.getLogger('DemosTask') def getInstance(task_context): return DemosTask(task_context) class DemosTask(AbstractTask): """DemosTask""" action_recorder = None def __init__(self, ctx): super(DemosTask, self).__init__(ctx) pass def rumCommand(self, cmd, options):
#coding=utf-8 import os import os.path import xlrd import json import shutil import yaml from rtool import utils from rtool.tasks import BasePreAction from rtool.projects.demos.extraCfg import genExtraConfigs as GEC from rtool.tasks import parse_config_files as pcfg from rtool.tasks import action_dispatch as ad logger = utils.getLogger('Task_Pre_Action_Config_Parse') dec_options = {} class PresActionConfigParse(BasePreAction.BasePreAction): """PresActionConfigParse封装了在框架中Actions执行之前需要做的操作,主要负责处理额外的配置解析""" def __init__(self, options): super(PresActionConfigParse, self).__init__(options) dec_options = options # @BasePreAction.preProcess(options = dec_options) def go(self): # 有额外的配置信息需要添加至extraCfg中使用以下代码 if not pcfg.task_settings.has_key('extraCfg'): pcfg.task_settings['extraCfg'] = {} # GEC.genExtraCfgs(pcfg.task_settings['extraCfg']) # 框架自身需要的前处理操作 BasePreAction.pre_process(self.options)
import time import rtool.tasks.BaseActionDispatch as BaseActionDispatch from rtool.tasks.BaseLocalChange import BaseLocalChange from rtool.tasks import BaseSync as BS from time import time global_cfg_dict = {} task_settings = {} white_dict = {} # img_in_csd_list = [] # img_in_csd_dict = {} old_global_cfg_dict = {} ncfg_dict = {} run_dict = {} logger = mut.getLogger('Task_parse_config') def modifyPathWithPattern(pattern, path_part, path): # print "modifyPathWithPattern" + pattern+ " "+path_part # print path if pattern in path: return path.replace(pattern, path_part) return path class BaseParseConfigs(object): """docstring for BaseParseConfigs""" def __init__(self, options): super(BaseParseConfigs, self).__init__() self._executeActionFunc = None
#coding=utf-8 import sys import requests import hashlib from rtool import utils from rtool.tasks.Base import Base logger = utils.getLogger("BaseStatus") class BaseStatus(Base): """docstring for BaseStatus""" def __init__(self,options): super(BaseStatus, self).__init__(options) self.options = options self.key = "" self.url = "" def doAPIRequest(self,tid,status): game = self.options['game'] token_str = tid+status+game+self.key md5 = hashlib.md5() md5.update(token_str.encode('utf-8')) token = md5.hexdigest() payload={} payload['id'] = tid payload['status'] = status payload['game'] = game payload['token'] = token r = requests.get(self.url,params=payload) print(r.status_code) print(r.content)
import subprocess import sys import tempfile import plistlib import zlib import struct from time import sleep sys.path.append(os.path.split(os.path.realpath(__file__))[0]) import rtool.taskplugin.plugin.MultiProcessRunner as MultiProcessRunner import rtool.utils as utils from rtool.taskplugin.plugin.TPTask import TPTask import rtool.taskplugin.plugin.TPCache2 as TPCache2 logger = utils.getLogger('EncryptImage') def run_with_configs(configs, tp=None): logger.debug("Executing Image Compress Action") apaction = EncryptImage() apaction.go(configs, tp) pass def clean_output(configs): pass class EncryptImage:
#coding:utf-8 import os import os.path import xlrd import json import shutil from . import extraCfg from rtool import utils logger = utils.getLogger('GenAnimIdleList') class GenAnimIdleList(extraCfg.ExtraCfg): """docstring for GenAnimIdleList""" def __init__(self, arg): super(GenAnimIdleList, self).__init__(arg) self.idle_name_list = [] def getIdleNameList(self, input_file_name): xls_file = xlrd.open_workbook(input_file_name) input_table = xls_file.sheets()[0] for i in range(input_table.ncols): value = input_table.cell_value(1, i) if value == "pinyinID[string]": for j in range(2, input_table.nrows): self.idle_name_list.append( str(input_table.cell_value(j, i))) pass def getAllIdleName(self, item_config_path=''):
import sys import json import copy import shutil import plistlib as pl import rtool.pcutils.md5sum_of_path as mput from xml.etree import ElementTree import plistlib from rtool.tasks import BaseSync as BS import rtool.tasks.BaseLocalChange as BLC import rtool.tasks.BaseDecorate as BD import rtool.tasks.BaseParseConfigs as BPC output_to_temp = {} logger = utils.getLogger('Task_post_action') #Function:缓存记录上次运行结果的md5 #param:上次配置, global_cfg_dict, #output: def calMD5forOutputPathAndPersisted(res_root, global_cfg_dict, target, action_recorder): if global_cfg_dict: for action_type in global_cfg_dict[target]: configs = global_cfg_dict[target][action_type] for config in configs: # print config['output-dir'] md5sum = mput.md5sum_of_path(config['input']) if md5sum:
#coding=utf-8 import yaml import os import os.path import shutil import json import subprocess import sys sys.path.append(os.path.split(os.path.realpath(__file__))[0]) import rtool.taskplugin.plugin.MultiProcessRunner as MultiProcessRunner import rtool.utils as utils logger = utils.getLogger('CopyRes') def run(): logger.debug("CopyRes") pass def run_with_configs(configs, tp=None): logger.debug("Executing CopyRes") apaction = CopyResAction() apaction.go(configs) pass def safeRemoveDir(dir_path): if os.path.exists(dir_path): shutil.rmtree(dir_path) pass
import sys import tempfile import plistlib import zlib import struct from time import sleep sys.path.append(os.path.split(os.path.realpath(__file__))[0]) import rtool.taskplugin.plugin.MultiProcessRunner as MultiProcessRunner import rtool.utils as utils import rtool.taskplugin.plugin.TPtemplates as TPtemplates from rtool.taskplugin.plugin.TPTask import TPTask from rtool.taskplugin.plugin.MaliTask import MaliTask import rtool.taskplugin.plugin.TPCache2 as TPCache2 logger = utils.getLogger('Compresser') def run(): logger.debug("Executing Image Compress Action") pass def run_with_configs(configs, tp=None): logger.debug("Executing Image Compress Action") apaction = CompressAction() apaction.go(configs, tp) pass def safeRemove(path):
def setLogName(self, name): self._logger = utils.getLogger(name)
#coding=utf-8 #coding=utf-8 import yaml import os import os.path import shutil import json import subprocess import sys sys.path.append(os.path.split(os.path.realpath(__file__))[0]) import MultiProcessRunner import rtool.utils as utils logger = utils.getLogger('SyncFilesAction') def run(): logger.debug("SyncFilesAction") pass def run_with_configs(configs, tp=None): logger.debug("Executing SyncFilesAction") apaction = SyncFilesAction() apaction.go(configs) pass def safeRemoveDir(dir_path): if os.path.exists(dir_path): shutil.rmtree(dir_path)
import rtool.utils as mut import os import os.path import sys import json import imp import rtool.pcutils.md5sum_of_path as mput import rtool.res.TPCache2 as TPCache2 import rtool.res.TPTask as TPTask import rtool.tasks.BaseSync as BS import rtool.tasks.BaseStatus as BStatus import rtool.taskplugin.plugin.TPDirFormater as TPDF logger = mut.getLogger('Task') class TaskContext(object): def __init__(self): self.taskName = "" # self.logger = mut.getLogger(self.taskName) self.settings = None self.options = None pass def getLogger(self): return logger def getEnv(self, name): # get environment variable with `name`
#coding=utf-8 import os import json import plistlib from multiprocessing import Process, Manager, Lock import shutil import sys sys.path.append(os.path.split(os.path.realpath(__file__))[0]) import rtool.utils as utils import rtool.res.action_binaryJson as AJ logger = utils.getLogger('Xml2Mclib') def run(): logger.debug("Xml2Mclib") pass def run_with_configs(configs, tp=None): logger.debug("Executing Xml2Mclib") apaction = Xml2Mclib() apaction.go(configs) pass def safeRemoveDir(dir_path): if os.path.exists(dir_path): shutil.rmtree(dir_path) pass
#coding=utf-8 import yaml import os import os.path import shutil import json import subprocess import sys sys.path.append(os.path.split(os.path.realpath(__file__))[0]) import rtool.taskplugin.plugin.MultiProcessRunner as MultiProcessRunner import rtool.utils as utils logger = utils.getLogger('CSD2CSB') def run(): logger.debug("Executing CsdPublish") pass def run_with_configs(configs, tp=None): logger.debug("Executing CsdPublish") # os.system('mtl ccs') apaction = CsdPublishAction() apaction.pack(configs) pass def clean_output(configs): default_output_path = configs["output-root"]
import os.path import shutil import json import subprocess import tempfile import sys import plistlib sys.path.append(os.path.split(os.path.realpath(__file__))[0]) import rtool.taskplugin.plugin.MultiProcessRunner import rtool.utils as utils import rtool.taskplugin.plugin.TPCache2 as TPCache2 from rtool.taskplugin.plugin.TPTask import TPTask from rtool.taskplugin.plugin.pcutils.remove_smartupdate import remove_smartupdate_from_plist_or_json from rtool.taskplugin.plugin.pcutils.remove_smartupdate import remove_smartupdate_from_plist_or_json_then_rename logger = utils.getLogger('AnimPacker') logger.setLevel("DEBUG") def relativePath(base_path,path): if not base_path == path: return path.replace(base_path+os.path.sep,'') else: return "" pass def run(): print("AnimPacker") pass def run_with_configs(configs,tp=None): logger.debug("Executing NAnimPacker")
import os import os.path import shutil import json import subprocess import tempfile import sys import plistlib sys.path.append(os.path.split(os.path.realpath(__file__))[0]) import MultiProcessRunner import rtool.utils as utils import TPCache2 from TPTask import TPTask from pcutils.remove_smartupdate import remove_smartupdate_from_plist_or_json logger = utils.getLogger('TexPacker') def relativePath(base_path, path): if not base_path == path: return path.replace(base_path + os.path.sep, '') else: return "" pass def run(): print "TexPacker" pass