def read_json(self, encoding="utf8", flexible=True, leaves=True):
content = self.read(encoding=encoding)
value = get_module("mo_json").json2value(content, flexible=flexible, leaves=leaves)
abspath = self.abspath
if os.sep == "\\":
abspath = "/" + abspath.replace(os.sep, "/")
return get_module("mo_json_config").expand(value, "file://" + abspath)
def read_json(self, encoding="utf8", flexible=True, leaves=True):
content = self.read(encoding=encoding)
value = get_module(u"mo_json").json2value(content,
flexible=flexible,
leaves=leaves)
abspath = self.abspath
if os.sep == "\\":
abspath = "/" + abspath.replace(os.sep, "/")
return get_module("mo_json_config").expand(value, "file://" + abspath)
def read(self, encoding="utf8"):
with open(self._filename, "rb") as f:
content = f.read().decode(encoding)
if self.key:
return get_module("mo_math.crypto").decrypt(content, self.key)
else:
return content
def read(self, encoding="utf8"):
with open(self._filename, "rb") as f:
content = f.read().decode(encoding)
if self.key:
return get_module(u"mo_math.crypto").decrypt(content, self.key)
else:
return content
def write(self, data):
if not self.parent.exists:
self.parent.create()
with open(self._filename, "wb") as f:
if isinstance(data, list) and self.key:
Log.error(
u"list of data and keys are not supported, encrypt before sending to file"
)
if isinstance(data, list):
pass
elif isinstance(data, (binary_type, text_type)):
data = [data]
elif hasattr(data, "__iter__"):
pass
for d in data:
if not isinstance(d, text_type):
Log.error(u"Expecting unicode data only")
if self.key:
f.write(
get_module(u"crypto").encrypt(d,
self.key).encode("utf8"))
else:
f.write(d.encode("utf8"))
def __new__(cls, value=None, **kwargs):
output = object.__new__(cls)
if value == None:
if kwargs:
output.milli = datetime.timedelta(**kwargs).total_seconds() * 1000
output.month = 0
return output
else:
return None
if Math.is_number(value):
output._milli = float(value) * 1000
output.month = 0
return output
elif isinstance(value, text_type):
return parse(value)
elif isinstance(value, Duration):
output.milli = value.milli
output.month = value.month
return output
elif isinstance(value, float) and Math.is_nan(value):
return None
else:
from mo_logs import Log
Log.error("Do not know type of object (" + get_module("mo_json").value2json(value) + ")of to make a Duration")
def decrypt(data, _key):
"""
ACCEPT BYTES -> UTF8 -> JSON -> {"salt":s, "length":l, "data":d}
"""
# Key and iv have not been generated or provided, bail out
if _key is None:
Log.error("Expecting a key")
_input = get_module("mo_json").json2value(data.decode('utf8'), leaves=False, flexible=False)
# Initialize encryption using key and iv
key_expander_256 = key_expander.KeyExpander(256)
expanded_key = key_expander_256.expand(_key)
aes_cipher_256 = aes_cipher.AESCipher(expanded_key)
aes_cbc_256 = cbc_mode.CBCMode(aes_cipher_256, 16)
aes_cbc_256.set_iv(base642bytearray(_input.salt))
raw = base642bytearray(_input.data)
out_data = bytearray()
for _, e in _groupby16(raw):
out_data.extend(aes_cbc_256.decrypt_block(e))
if _input.encoding:
return binary_type(out_data[:_input.length:]).decode(_input.encoding)
else:
return binary_type(out_data[:_input.length:])
def __new__(cls, value=None, **kwargs):
output = object.__new__(cls)
if value == None:
if kwargs:
output.milli = datetime.timedelta(
**kwargs).total_seconds() * 1000
output.month = 0
return output
else:
return None
if is_number(value):
output._milli = float(value) * 1000
output.month = 0
return output
elif is_text(value):
return parse(value)
elif isinstance(value, Duration):
output.milli = value.milli
output.month = value.month
return output
elif isinstance(value, float) and is_nan(value):
return None
else:
from mo_logs import Log
Log.error("Do not know type of object (" +
get_module("mo_json").value2json(value) +
")of to make a Duration")
def decrypt(data, _key):
"""
ACCEPT BYTES -> UTF8 -> JSON -> {"salt":s, "length":l, "data":d}
"""
# Key and iv have not been generated or provided, bail out
if _key is None:
Log.error("Expecting a key")
_input = get_module("mo_json").json2value(data.decode("utf8"),
leaves=False,
flexible=False)
# Initialize encryption using key and iv
key_expander_256 = key_expander.KeyExpander(256)
expanded_key = key_expander_256.expand(_key)
aes_cipher_256 = aes_cipher.AESCipher(expanded_key)
aes_cbc_256 = cbc_mode.CBCMode(aes_cipher_256, 16)
aes_cbc_256.set_iv(base642bytes(_input.salt))
raw = base642bytes(_input.data)
out_data = bytearray()
for _, e in _groupby16(raw):
out_data.extend(aes_cbc_256.decrypt_block(e))
if _input.encoding:
return binary_type(out_data[:_input.length:]).decode(_input.encoding)
else:
return binary_type(out_data[:_input.length:])
def write_bytes(self, content):
if not self.parent.exists:
self.parent.create()
with open(self._filename, "wb") as f:
if self.key:
f.write(get_module("mo_math.crypto").encrypt(content, self.key))
else:
f.write(content)
def write_bytes(self, content):
if not self.parent.exists:
self.parent.create()
with open(self._filename, "wb") as f:
if self.key:
f.write(get_module("mo_math.crypto").encrypt(content, self.key))
else:
f.write(content)
def read(self, encoding="utf8"):
"""
:param encoding:
:return:
"""
with open(self._filename, "rb") as f:
if self.key:
return get_module("mo_math.crypto").decrypt(f.read(), self.key)
else:
content = f.read().decode(encoding)
return content
def read_bytes(self):
try:
if not self.parent.exists:
self.parent.create()
with open(self._filename, "rb") as f:
if self.key:
return get_module("mo_math.crypto").decrypt(f.read(), self.key)
else:
return f.read()
except Exception as e:
Log.error(u"Problem reading file {{filename}}", filename=self.abspath, cause=e)
def read(self, encoding="utf8"):
"""
:param encoding:
:return:
"""
with open(self._filename, "rb") as f:
if self.key:
return get_module("mo_math.crypto").decrypt(f.read(), self.key)
else:
content = f.read().decode(encoding)
return content
def read_bytes(self):
try:
if not self.parent.exists:
self.parent.create()
with open(self._filename, "rb") as f:
if self.key:
return get_module("mo_math.crypto").decrypt(f.read(), self.key)
else:
return f.read()
except Exception as e:
Log.error(u"Problem reading file {{filename}}", filename=self.abspath, cause=e)
def _late_import():
global _json_encoder
global _Log
global _Except
global _Duration
_json_encoder = get_module("mo_json.encoder").json_encoder
from mo_logs import Log as _Log
from mo_logs.exceptions import Except as _Except
from mo_times.durations import Duration as _Duration
_ = _json_encoder
_ = _Log
_ = _Except
_ = _Duration
def encrypt(text, _key, salt=None):
"""
RETURN {"salt":s, "length":l, "data":d} -> JSON -> UTF8
"""
if is_text(text):
encoding = "utf8"
data = bytearray(text.encode("utf8"))
elif is_binary(text):
encoding = None
if PY2:
data = bytearray(text)
else:
data = text
if _key is None:
Log.error("Expecting a key")
if is_binary(_key):
_key = bytearray(_key)
if salt is None:
salt = crypto.bytes(16)
# Initialize encryption using key and iv
key_expander_256 = key_expander.KeyExpander(256)
expanded_key = key_expander_256.expand(_key)
aes_cipher_256 = aes_cipher.AESCipher(expanded_key)
aes_cbc_256 = cbc_mode.CBCMode(aes_cipher_256, 16)
aes_cbc_256.set_iv(salt)
output = Data()
output.type = "AES256"
output.salt = bytes2base64(salt)
output.length = len(data)
output.encoding = encoding
encrypted = bytearray()
for _, d in _groupby16(data):
encrypted.extend(aes_cbc_256.encrypt_block(d))
output.data = bytes2base64(encrypted)
json = get_module("mo_json").value2json(output, pretty=True).encode("utf8")
if DEBUG:
test = decrypt(json, _key)
if test != text:
Log.error("problem with encryption")
return json
def encrypt(text, _key, salt=None):
"""
RETURN {"salt":s, "length":l, "data":d} -> JSON -> UTF8
"""
if is_text(text):
encoding = 'utf8'
data = bytearray(text.encode("utf8"))
elif is_binary(text):
encoding = None
if PY2:
data = bytearray(text)
else:
data = text
if _key is None:
Log.error("Expecting a key")
if is_binary(_key):
_key = bytearray(_key)
if salt is None:
salt = Random.bytes(16)
# Initialize encryption using key and iv
key_expander_256 = key_expander.KeyExpander(256)
expanded_key = key_expander_256.expand(_key)
aes_cipher_256 = aes_cipher.AESCipher(expanded_key)
aes_cbc_256 = cbc_mode.CBCMode(aes_cipher_256, 16)
aes_cbc_256.set_iv(salt)
output = Data()
output.type = "AES256"
output.salt = bytes2base64(salt)
output.length = len(data)
output.encoding = encoding
encrypted = bytearray()
for _, d in _groupby16(data):
encrypted.extend(aes_cbc_256.encrypt_block(d))
output.data = bytes2base64(encrypted)
json = get_module("mo_json").value2json(output, pretty=True).encode('utf8')
if DEBUG:
test = decrypt(json, _key)
if test != text:
Log.error("problem with encryption")
return json
def _late_import():
global _json_encoder
global _Log
global _Except
global _Duration
try:
_json_encoder = get_module("mo_json.encoder").json_encoder
except Exception:
_json_encoder = lambda value, pretty: _json.dumps(value)
from mo_logs import Log as _Log
from mo_logs.exceptions import Except as _Except
from mo_times.durations import Duration as _Duration
_ = _json_encoder
_ = _Log
_ = _Except
_ = _Duration
def _late_import():
global _json_encoder
global _Log
global _Except
global _Duration
try:
_json_encoder = get_module("mo_json.encoder").json_encoder
except Exception:
_json_encoder = lambda value, pretty: _json.dumps(value)
from mo_logs import Log as _Log
from mo_logs.exceptions import Except as _Except
from mo_times.durations import Duration as _Duration
_ = _json_encoder
_ = _Log
_ = _Except
_ = _Duration
def write(self, data):
if not self.parent.exists:
self.parent.create()
with open(self._filename, "wb") as f:
if isinstance(data, list) and self.key:
Log.error("list of data and keys are not supported, encrypt before sending to file")
if isinstance(data, list):
pass
elif isinstance(data, basestring):
data=[data]
elif hasattr(data, "__iter__"):
pass
for d in data:
if not isinstance(d, unicode):
Log.error("Expecting unicode data only")
if self.key:
f.write(get_module("crypto").encrypt(d, self.key).encode("utf8"))
else:
f.write(d.encode("utf8"))
def encrypt(text, _key, salt=None):
"""
RETURN JSON OF ENCRYPTED DATA {"salt":s, "length":l, "data":d}
"""
if not isinstance(text, unicode):
Log.error("only unicode is encrypted")
if _key is None:
Log.error("Expecting a key")
if isinstance(_key, str):
_key = bytearray(_key)
if salt is None:
salt = Random.bytes(16)
data = bytearray(text.encode("utf8"))
# Initialize encryption using key and iv
key_expander_256 = key_expander.KeyExpander(256)
expanded_key = key_expander_256.expand(_key)
aes_cipher_256 = aes_cipher.AESCipher(expanded_key)
aes_cbc_256 = cbc_mode.CBCMode(aes_cipher_256, 16)
aes_cbc_256.set_iv(salt)
output = Data()
output.type = "AES256"
output.salt = bytes2base64(salt)
output.length = len(data)
encrypted = bytearray()
for _, d in _groupby16(data):
encrypted.extend(aes_cbc_256.encrypt_block(d))
output.data = bytes2base64(encrypted)
json = get_module("mo_json").value2json(output)
if DEBUG:
test = decrypt(json, _key)
if test != text:
Log.error("problem with encryption")
return json
def encrypt(text, _key, salt=None):
"""
RETURN JSON OF ENCRYPTED DATA {"salt":s, "length":l, "data":d}
"""
if not isinstance(text, text_type):
Log.error("only unicode is encrypted")
if _key is None:
Log.error("Expecting a key")
if isinstance(_key, str):
_key = bytearray(_key)
if salt is None:
salt = Random.bytes(16)
data = bytearray(text.encode("utf8"))
# Initialize encryption using key and iv
key_expander_256 = key_expander.KeyExpander(256)
expanded_key = key_expander_256.expand(_key)
aes_cipher_256 = aes_cipher.AESCipher(expanded_key)
aes_cbc_256 = cbc_mode.CBCMode(aes_cipher_256, 16)
aes_cbc_256.set_iv(salt)
output = Data()
output.type = "AES256"
output.salt = bytes2base64(salt)
output.length = len(data)
encrypted = bytearray()
for _, d in _groupby16(data):
encrypted.extend(aes_cbc_256.encrypt_block(d))
output.data = bytes2base64(encrypted)
json = get_module("mo_json").value2json(output)
if DEBUG:
test = decrypt(json, _key)
if test != text:
Log.error("problem with encryption")
return json
def _late_import():
global _json_encoder
global _Log
global _Except
global _Duration
try:
_json_encoder = get_module("mo_json.encoder").json_encoder
except Exception:
_json_encoder = lambda value, pretty: _json.dumps(value)
from mo_logs import Log as _Log
from mo_logs.exceptions import Except as _Except
try:
from mo_times.durations import Duration as _Duration
except Exception as e:
_Duration = NullType
_Log.warning("It would be nice to pip install mo-times", cause=e)
_ = _json_encoder
_ = _Log
_ = _Except
_ = _Duration
def decrypt(data, _key):
"""
ACCEPT JSON OF ENCRYPTED DATA {"salt":s, "length":l, "data":d}
"""
# Key and iv have not been generated or provided, bail out
if _key is None:
Log.error("Expecting a key")
_input = get_module("mo_json").json2value(data)
# Initialize encryption using key and iv
key_expander_256 = key_expander.KeyExpander(256)
expanded_key = key_expander_256.expand(_key)
aes_cipher_256 = aes_cipher.AESCipher(expanded_key)
aes_cbc_256 = cbc_mode.CBCMode(aes_cipher_256, 16)
aes_cbc_256.set_iv(base642bytearray(_input.salt))
raw = base642bytearray(_input.data)
out_data = bytearray()
for _, e in _groupby16(raw):
out_data.extend(aes_cbc_256.decrypt_block(e))
return str(out_data[:_input.length:]).decode("utf8")
def decrypt(data, _key):
"""
ACCEPT JSON OF ENCRYPTED DATA {"salt":s, "length":l, "data":d}
"""
# Key and iv have not been generated or provided, bail out
if _key is None:
Log.error("Expecting a key")
_input = get_module("mo_json").json2value(data)
# Initialize encryption using key and iv
key_expander_256 = key_expander.KeyExpander(256)
expanded_key = key_expander_256.expand(_key)
aes_cipher_256 = aes_cipher.AESCipher(expanded_key)
aes_cbc_256 = cbc_mode.CBCMode(aes_cipher_256, 16)
aes_cbc_256.set_iv(base642bytearray(_input.salt))
raw = base642bytearray(_input.data)
out_data = bytearray()
for _, e in _groupby16(raw):
out_data.extend(aes_cbc_256.decrypt_block(e))
return str(out_data[:_input.length:]).decode("utf8")
from __future__ import unicode_literals
import os
import subprocess
from types import NoneType
from mo_dots import set_default, unwrap, get_module, NullType
from mo_logs import Log, strings
from mo_logs.exceptions import Except
from mo_threads.lock import Lock
from mo_threads.queues import Queue
from mo_threads.signal import Signal
from mo_threads.threads import Thread, THREAD_STOP
string2quote = get_module("mo_json").quote
DEBUG = False
class Process(object):
def __init__(self,
name,
params,
cwd=None,
env=None,
debug=False,
shell=False,
bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " +
string2quote(name))
def __str__(self):
return "Matrix " + get_module("mo_json").value2json(
self.dims) + ": " + str(self.cube)
def __str__(self):
return "Matrix " + get_module("mo_json").value2json(self.dims) + ": " + str(self.cube)
