コード例 #1
0
def loadFromFile(filePath:str, scmgr = None, structureTypeName:str = None) -> FlexObject:
	assert isinstance(filePath, str)

	if bJSONCfgHelperAvailable:
		if scmgr or structureTypeName:
			Assert.isIn(scmgr.__class__.__name__, [ "StructureCheckerManager", "jk_jsoncfghelper2.StructureCheckerManager" ])
			Assert.isInstance(structureTypeName, str)

		with open(filePath, "r") as f:
			data = json.load(f)
		assert isinstance(data, dict)

		if scmgr or structureTypeName:
			checker = scmgr.getE(structureTypeName)
			if checker.checkB(scmgr, data):
				return FlexObject(data)
			else:
				raise Exception("Data does not match type " + repr(structureTypeName))	# TODO
		else:
			return FlexObject(data)

	else:
		if (scmgr is not None) or (structureTypeName is not None):
			raise Exception("As module jk_jsoncfghelper2 is not installed, scmgr and structureTypeName must noe None!")

		with open(filePath, "r") as f:
			data = json.load(f)
		assert isinstance(data, dict)

		return FlexObject(data)
コード例 #2
0
    def __perform_calcDiskSpaceRequired(
            self, bd2: BD2,
            backupTasks: typing.List[AbstractThaniyaTask]) -> int:
        with ProcessingContext(text="Calculating disk space required",
                               bd2=bd2,
                               bMeasureDuration=True,
                               statsDurationKey="d0_calcDiskSpace") as ctx:
            nExpectedBytesToWrite = 0
            for job in backupTasks:
                #assert isinstance(job, AbstractThaniyaTask)
                Assert.isInstance(job, AbstractThaniyaTask)

                nestedCtx = ctx.descend(job.logMessageCalculateSpaceRequired)
                with nestedCtx.log as nestedLog:
                    nExpectedBytesToWrite += job.calculateSpaceRequired(
                        nestedCtx)

            ctx.log.info("Estimated total size of backup: " +
                         jk_utils.formatBytes(nExpectedBytesToWrite))

            bd2.statsContainer.setValue("expectedBytesToWrite",
                                        nExpectedBytesToWrite)

            # ----

            ctx.log.notice("Done.")

        return nExpectedBytesToWrite
コード例 #3
0
	def __init__(self, tp:AbstractTokenPattern, emitName:str = None, emitValue = None):
		Assert.isInstance(tp, AbstractTokenPattern)
		if emitName is not None:
			Assert.isInstance(emitName, str)
		self.__tp = tp
		self.emitName = emitName
		self.emitValue = emitValue
コード例 #4
0
 def __getAny(d: dict, key1: str, key2: str, typeOrTypes):
     v = d.get(key1)
     if v is None:
         v = d.get(key2)
     if v is not None:
         Assert.isInstance(v, typeOrTypes)
     return v
コード例 #5
0
    def __deserializeTable(textConverterMgr: TextConverterManager,
                           jobj: dict) -> TokenizingTable:
        Assert.isInstance(jobj, dict)

        jrows = JSONLoader.__getE(jobj, "on", (list, tuple))

        rows = []
        if jrows is not None:
            for i in range(0, len(jrows)):
                rows.append(
                    JSONLoader.__deserializeRow(
                        textConverterMgr, JSONLoader.__getE(jrows, i, dict)))

        tableID = JSONLoader.__getE(jobj, "tableID", int)
        tableName = JSONLoader.__getE(jobj, "tableName", str)

        table = TokenizingTable(tableID)
        table.tableName = tableName
        for r in rows:
            table.addPatternRow(r.pattern, r.actions)

        jtemp = JSONLoader.__getE(jobj, "onOther", (list, tuple))
        if jtemp is not None:
            table.setOther(
                JSONLoader.__deserializeActions(textConverterMgr, jtemp))

        jtemp = JSONLoader.__getE(jobj, "onEOS", (list, tuple))
        if jtemp is not None:
            table.setEOS(
                JSONLoader.__deserializeActions(textConverterMgr, jtemp))

        return table
コード例 #6
0
 def __get(d: Union[dict, list, tuple], key: Union[str, int], typeOrTypes):
     if isinstance(d, dict):
         v = d.get(key)
     else:
         v = d[key]
     if v is not None:
         Assert.isInstance(v, typeOrTypes)
     return v
コード例 #7
0
def _compile_float(scmgr: StructureCheckerManager, x: JDef):
    Assert.isEqual(x.dataType, "float")
    #assert x.dataType == "float"

    return FloatValueChecker(scmgr,
                             minValue=x.minValue,
                             maxValue=x.maxValue,
                             required=x.required,
                             nullable=x.nullable)
コード例 #8
0
def _compile_int(scmgr: StructureCheckerManager, x: JDef):
    Assert.isIn(x.dataType, ["int", "integer"])
    #assert x.dataType in [ "int", "integer" ]

    return IntValueChecker(scmgr,
                           minValue=x.minValue,
                           maxValue=x.maxValue,
                           required=x.required,
                           nullable=x.nullable)
コード例 #9
0
def _compile_anydict(scmgr: StructureCheckerManager, x: JDef):
    Assert.isIn(x.dataType, ["dict", "dictionary"])
    #assert x.dataType in [ "dict", "dictionary" ]

    return AnyDictionaryValueChecker(
        scmgr,
        required=x.required,
        allowedElementTypes=__compile_allowedElements(scmgr, x.elementTypes),
        nullable=x.nullable)
コード例 #10
0
 def __init__(self, *args, emitName: str = None, emitValue=None):
     assert len(args) > 0
     for a in args:
         Assert.isInstance(a, AbstractTokenPattern)
     self.__tokenPatterns = args
     if emitName is not None:
         Assert.isInstance(emitName, str)
     self.emitName = emitName
     self.emitValue = emitValue
コード例 #11
0
	def register(self, name:str, checker:AbstractValueChecker):
		Assert.isInstance(name, str)
		#assert isinstance(name, str)
		Assert.isInstance(checker, AbstractValueChecker)
		#assert isinstance(checker, AbstractValueChecker)

		self.__types[name] = checker

		return checker
コード例 #12
0
def _compile_str(scmgr: StructureCheckerManager, x: JDef):
    Assert.isIn(x.dataType, ["str", "string"])
    #assert x.dataType in [ "str", "string" ]

    return StringValueChecker(scmgr,
                              minLength=x.minLength,
                              maxLength=x.maxLength,
                              allowedValues=x.allowedValues,
                              required=x.required,
                              nullable=x.nullable)
コード例 #13
0
 def __getAnyE(d: dict, key1: str, key2: str, typeOrTypes):
     v = d.get(key1)
     if v is None:
         v = d.get(key2)
     if v is None:
         raise Exception("None of these keys exist: " + repr(key1) + ", " +
                         repr(key2))
     if v is not None:
         Assert.isInstance(v, typeOrTypes)
     return v
コード例 #14
0
 def __getE(d: Union[dict, list, tuple], key: Union[str, int], typeOrTypes):
     if isinstance(d, dict):
         v = d.get(key)
     else:
         v = d[key]
     if v is None:
         raise Exception("No such key: " + repr(key))
     if v is not None:
         Assert.isInstance(v, typeOrTypes)
     return v
コード例 #15
0
def _compile_list(scmgr: StructureCheckerManager, x: JDef):
    Assert.isEqual(x.dataType, "list")
    #assert x.dataType == "list"

    return ListValueChecker(scmgr,
                            minLength=x.minLength,
                            maxLength=x.maxLength,
                            allowedElementTypes=__compile_allowedElements(
                                scmgr, x.elementTypes),
                            required=x.required,
                            nullable=x.nullable)
コード例 #16
0
 def __init__(self, *args, emitName: str = None, emitValue=None):
     assert len(args) > 0
     for a in args:
         Assert.isInstance(a, AbstractTokenPattern)
         if isinstance(a, TPOptional):
             raise Exception(
                 "An TPOptional element should not be part of a TPAlt element as this breaks matching!"
             )
     self.__tokenPatterns = args
     self.emitName = emitName
     self.emitValue = emitValue
コード例 #17
0
 def __init__(self,
              tp: AbstractTokenPattern,
              delimiterPattern: AbstractTokenPattern,
              emitName: str = None,
              emitValue=None):
     assert isinstance(tp, AbstractTokenPattern)
     if delimiterPattern:
         assert isinstance(delimiterPattern, AbstractTokenPattern)
     if emitName is not None:
         Assert.isInstance(emitName, str)
     self.__tokenPattern = tp
     self.__delimiterPattern = delimiterPattern
     self.emitName = emitName
     self.emitValue = emitValue
コード例 #18
0
 def __init__(self,
              type_: str,
              text_: str = None,
              bIgnoreCase: bool = False,
              emitName: str = None,
              emitValue=None):
     self.type = type_
     if text_ is None:
         self.text = None
         self.bIgnoreCase = False
     else:
         self.text = text_.lower() if bIgnoreCase else text_
         self.bIgnoreCase = bIgnoreCase
     if emitName is not None:
         Assert.isInstance(emitName, str)
     self.emitName = emitName
     self.emitValue = emitValue
コード例 #19
0
    def __perform_backup(self, bd2: BD2,
                         backupTasks: typing.List[AbstractThaniyaTask]):

        # NOTE: we need to access this context later as it calculates the duration and we need this information separately to log it.
        processingContext = ProcessingContext(text="Writing the backup data",
                                              bd2=bd2,
                                              bMeasureDuration=True,
                                              statsDurationKey="d2_backup")

        with processingContext as ctx:

            for job in backupTasks:
                Assert.isInstance(job, AbstractThaniyaTask)

                with ctx.descend(job.logMessagePerformBackup) as nestedCtx:
                    job.performBackup(nestedCtx)

            ctx.log.notice("All backup tasks completed.")

            # calculate statistics

            with ctx.log.descend(
                    "Calculating size of backup performed ...") as nestedLog:
                nTotalBytesWritten = jk_utils.fsutils.getFolderSize(
                    bd2.effectiveTargetDirPath)

            fDuration = processingContext.duration
            if (nTotalBytesWritten > 0) and (fDuration > 0):
                fAvgWritingSpeed = nTotalBytesWritten / fDuration
                sAvgWritingSpeed = jk_utils.formatBytesPerSecond(
                    fAvgWritingSpeed)
            else:
                fAvgWritingSpeed = None
                sAvgWritingSpeed = "n/a"

            ctx.log.info("Total bytes written: " +
                         jk_utils.formatBytes(nTotalBytesWritten))
            ctx.log.info("Average writing speed: " + sAvgWritingSpeed)

            bd2.statsContainer.setValue("totalBytesWritten",
                                        nTotalBytesWritten)
            bd2.statsContainer.setValue("avgWritingSpeed", fAvgWritingSpeed)

            # ----

            ctx.log.notice("Done.")
コード例 #20
0
	def connect(self) -> fabric.Connection:
		if self.__c is None:
			pwd = self.__passwordProvider(self.__hostName, self.__userName)
			config = fabric.Config(overrides={'sudo': {'password': pwd}})
			self.__c = fabric.Connection(self.__hostName, self.__userName, self.__port, config=config, connect_kwargs={"password": pwd})

			r = self.__c.run(
				self._encodeSSHCmdLine("/bin/echo", "foo  \"  \"  bar"),
				hide=True)
			Assert.isTrue(self.__c.is_connected)
			Assert.isEqual(r.exited, 0)
			Assert.isEqual(r.stderr, "")
			Assert.isEqual(r.stdout, "foo  \"  \"  bar\n")

		return self.__c
コード例 #21
0
import datetime

import jk_utils.datetime
from jk_utils.datatypes import *
from jk_testing import Assert








t = getTypeAsStr(123)
print(t)
Assert.isEqual(t, "int")

t = getTypeAsStr(3.14)
print(t)
Assert.isEqual(t, "float")

t = getTypeAsStr("abc")
print(t)
Assert.isEqual(t, "str")

t = getTypeAsStr(True)
print(t)
Assert.isEqual(t, "bool")

t = getTypeAsStr(False)
print(t)
コード例 #22
0
	def getE(self, name:str) -> AbstractValueChecker:
		Assert.isInstance(name, str)
		#assert isinstance(name, str)

		return self.__types[name]
コード例 #23
0
    def __init__(self, bd2, log: jk_logging.AbstractLogger):
        #Assert.isInstance(bd2, "BD2")								# extend Assert to support string class names
        Assert.isInstance(log, jk_logging.AbstractLogger)

        self.__bd2 = bd2
        self.__log = log
コード例 #24
0
#!/usr/bin/python3

from jk_utils.datetime import *
from jk_testing import Assert

d1 = D.now()
d2 = d1.nextDay().nextDay()
Assert.isEqual(d2 - d1, 2)
Assert.isEqual((d1.dayOfWeek + 2) % 7, d2.dayOfWeek)

d1 = D.now().startOfWeek()
d2 = d1.nextWeek()
Assert.isEqual(d2 - d1, 7)

d1 = D.now()
d2 = d1.nextWeek()
Assert.isTrue((d2 - d1) > 0)
Assert.isTrue((d2 - d1) <= 7)
Assert.isTrue(d2.dayOfWeek == 1)
Assert.isTrue((d2 - 1).dayOfWeek == 7)
Assert.isTrue(d2 >= d1)
Assert.isTrue(d1 <= d2)

d1 = D.now()
d2 = d1.nextWeek().previousWeek()
Assert.isTrue(d1 >= d2)

for d in dateRange(d1.startOfWeek(), d2.nextWeek()):
    print(d, "\t", d.toJSON())

Assert.isEqual(len(list(dateRange(d1.startOfWeek(), d2.nextWeek()))), 7)
コード例 #25
0
#!/usr/bin/python3

from jk_testing import Assert
from jk_trioinput import ConsoleInputHistory

h = ConsoleInputHistory()

h.append("123")
h.append("wieu")
h.append("LODJFKJHS")
h.resetCursor()

Assert.isEqual(h.prev(), "LODJFKJHS")
Assert.isEqual(h.prev(), "wieu")
Assert.isEqual(h.next(), "LODJFKJHS")
Assert.isEqual(h.prev(), "wieu")
Assert.isEqual(h.prev(), "123")
Assert.isEqual(h.next(), "wieu")
Assert.isEqual(h.prev(), "123")
Assert.isEqual(h.prev(), None)
Assert.isEqual(h.next(), "wieu")
Assert.isEqual(h.next(), "LODJFKJHS")
Assert.isEqual(h.next(), None)

h = ConsoleInputHistory()

h.append("123")
h.append("wieu")
h.append("LODJFKJHS")
h.resetCursor()
コード例 #26
0
def toV(value,
        dataType: str = None,
        defaultType: str = None) -> jk_flexdata.FlexObject:
    if dataType is None:
        # autodetect type
        if value is None:
            # try to accept the default type
            if defaultType is None:
                raise Exception("Value is None and default type is not set!")
            assert defaultType in [
                "int", "str", "bool", "float", "int[]", "str[]", "float[]",
                "tempc", "timestamputc", "timestamp", "duration", "bytes",
                "freq", "secsdiff"
            ]
            dataType = defaultType
        elif isinstance(value, bool):
            dataType = "bool"
        elif isinstance(value, int):
            dataType = "int"
        elif isinstance(value, float):
            dataType = "float"
        elif isinstance(value, str):
            dataType = "str"
        elif isinstance(value, list):
            nCountStr = 0
            nCountInt = 0
            nCountFloat = 0
            for item in value:
                if isinstance(item, float):
                    nCountFloat += 1
                elif isinstance(item, int):
                    nCountInt += 1
                elif isinstance(item, str):
                    nCountStr += 1
                else:
                    raise Exception("Unknown list item data type: " +
                                    repr(type(item)))
            if nCountInt == nCountFloat == nCountStr == 0:
                # assume it is a string lst
                dataType = "str[]"
            elif (nCountInt * nCountFloat !=
                  0) or (nCountInt * nCountStr !=
                         0) or (nCountFloat * nCountStr != 0):
                raise Exception("List with mixed item types!")
            else:
                if nCountFloat > 0:
                    dataType = "float[]"
                elif nCountInt > 0:
                    dataType = "int[]"
                else:
                    dataType = "str[]"
        else:
            raise Exception("Unknown data type: " + repr(type(value)))

    else:
        # type has been specified
        if value is None:
            # accept the type as it is
            pass
        elif dataType == "bool":
            Assert.isInstance(value, bool)
        elif dataType == "int":
            Assert.isInstance(value, int)
        elif dataType == "float":
            Assert.isInstance(value, float)
        elif dataType == "str":
            Assert.isInstance(value, str)
        elif dataType == "tempc":
            Assert.isInstance(value, (int, float))
        elif dataType == "timestamputc":
            if isinstance(value, datetime.datetime):
                value = value.timestamp()
            else:
                Assert.isInstance(value, (int, float))
        elif dataType == "timestamp":
            if isinstance(value, datetime.datetime):
                value = value.timestamp()
            else:
                Assert.isInstance(value, (int, float))
        elif dataType == "duration":
            Assert.isInstance(value, (int, float))
        elif dataType == "bytes":
            Assert.isInstance(value, int)
        elif dataType == "freq":
            Assert.isInstance(value, (int, float))
        elif dataType == "secsdiff":
            Assert.isInstance(value, (int, float))
        else:
            raise Exception("Invalid data type: " + repr(dataType))

    return jk_flexdata.FlexObject({"dt": dataType, "v": value})
コード例 #27
0
def _compile_bool(scmgr: StructureCheckerManager, x: JDef):
    Assert.isIn(x.dataType, ["bool", "boolean"])
    #assert x.dataType in [ "bool", "boolean" ]

    return BooleanValueChecker(scmgr, required=x.required, nullable=x.nullable)
コード例 #28
0
def _compile_null(scmgr: StructureCheckerManager, x: JDef):
    Assert.isEqual(x.dataType, "null")
    #assert x.dataType == "null"

    return NullValueChecker(scmgr, required=x.required)
コード例 #29
0
    ["|c|d|e", None, []],
    ["|c|*", int, 123, [
        "|c|d",
    ]],
    ["|c|*|d", None, []],
    ["|a|b", None, []],
]

for p in PATTERNS:
    print("NOW PROCESSING:", p)
    if len(p) == 3:
        spath, result = FlexDataSelector(p[0]).getOne(dataTree)
        print("\tspath = " + repr(spath))
        print("\tresult = " + repr(result))
        if p[1] is None:
            Assert.isNone(result)
            Assert.isNone(spath)
        else:
            Assert.isInstance(result, p[1])
            Assert.isInstance(spath, str)
            Assert.isIn(spath, p[2])
    elif len(p) == 4:
        spath, result = FlexDataSelector(p[0]).getOne(dataTree)
        if p[1] is None:
            Assert.isNone(spath)
            Assert.isNone(result)
        else:
            Assert.isInstance(result, p[1])
            Assert.isEqual(result, p[2])
            Assert.isInstance(spath, str)
            Assert.isIn(spath, p[3])
コード例 #30
0
TEST_VECTOR_1 = (
	(	"A1",	"/",		"/ab.cd",				"/ab.cd"			),
	(	"A2",	"/",		"ab.cd",				"/ab.cd"			),
	(	"A3",	"/xy/z/",	"ab.cd",				"/xy/z/ab.cd"		),
	(	"A4",	"/xy/z/",	"../ab.cd",				"/xy/ab.cd"			),
	(	"A5",	"/xy/z/",	".././ab.cd",			"/xy/ab.cd"			),
	(	"A6",	"/xy/z/",	".././../ab.cd",		"/ab.cd"			),
	(	"A7",	"/xy/z/",	".././../../ab.cd",		"/../ab.cd"			),
	(	"A8",	"/xy/z/",	"/a/ab.cd",				"/a/ab.cd"			),
	(	"A9",	"/xy/z/",	"/../ab.cd",			"/../ab.cd"			),
)

for label, mergeURL1, mergeURL2, validationURL in TEST_VECTOR_1:
	print(label)
	Assert.isEqual(
		mergeURLPaths(mergeURL1, mergeURL2),
		validationURL
	)


print()



urlCanonicalizer = URLCanonicalizer("https://www.xyz/startx/")


TEST_VECTOR_2 = (
	(	"B1",	"http://test.abc/def.yz",	"http://test.abc/def.yz",				),
	(	"B2",	"//test.abc/def.yz",		"https://test.abc/def.yz",				),
	(	"B3", 	"http:///def.yz",			"http://www.xyz/def.yz",				),
	(	"B4", 	"/def.yz",					"https://www.xyz/def.yz",				),