def canonicalize_env(env):
"""Windows requires that environment be dicts with bytes as keys and values
This function converts any unicode entries for Windows only, returning the
dictionary untouched in other environments.
Parameters
----------
env : dict
environment dictionary with unicode or bytes keys and values
Returns
-------
env : dict
Windows: environment dictionary with bytes keys and values
Other: untouched input ``env``
"""
if os.name != 'nt':
return env
# convert unicode to string for python 2
if not PY3:
from future.utils import bytes_to_native_str
out_env = {}
for key, val in env.items():
if not isinstance(key, bytes):
key = key.encode('utf-8')
if not isinstance(val, bytes):
val = val.encode('utf-8')
if not PY3:
key = bytes_to_native_str(key)
val = bytes_to_native_str(val)
out_env[key] = val
return out_env
def remote_advise_corrupt_share(self, share_type, storage_index, shnum,
reason):
# This is a remote API, I believe, so this has to be bytes for legacy
# protocol backwards compatibility reasons.
assert isinstance(share_type, bytes)
assert isinstance(reason, bytes), "%r is not bytes" % (reason, )
fileutil.make_dirs(self.corruption_advisory_dir)
now = time_format.iso_utc(sep="T")
si_s = si_b2a(storage_index)
# windows can't handle colons in the filename
fn = os.path.join(self.corruption_advisory_dir,
"%s--%s-%d" % (now, si_s, shnum)).replace(":", "")
with open(fn, "w") as f:
f.write("report: Share Corruption\n")
f.write("type: %s\n" % bytes_to_native_str(share_type))
f.write("storage_index: %s\n" % bytes_to_native_str(si_s))
f.write("share_number: %d\n" % shnum)
f.write("\n")
f.write(bytes_to_native_str(reason))
f.write("\n")
log.msg(format=("client claims corruption in (%(share_type)s) " +
"%(si)s-%(shnum)d: %(reason)s"),
share_type=share_type,
si=si_s,
shnum=shnum,
reason=reason,
level=log.SCARY,
umid="SGx2fA")
return None
def cols(self, size, current):
self.__initcheck()
ic = current.adapter.getCommunicator()
types = self.__types
names = self.__mea.colnames
descs = self.__descriptions
cols = []
for i in range(len(types)):
t = types[i]
if isbytes(t):
t = bytes_to_native_str(t)
n = names[i]
d = descs[i]
if isbytes(d):
d = bytes_to_native_str(d)
try:
col = ic.findObjectFactory(t).create(t)
col.name = n
col.description = d
col.setsize(size)
col.settable(self.__mea)
cols.append(col)
except:
msg = traceback.format_exc()
raise omero.ValidationException(
None, msg, "BAD COLUMN TYPE: %s for %s" % (t, n))
return cols
def _remove_uuid(proto):
if isinstance(proto, caffe2_pb2.NetDef):
for op in proto.op:
op.ClearField(bytes_to_native_str(b'uuid'))
elif isinstance(proto, caffe2_pb2.OperatorDef):
proto.ClearField(bytes_to_native_str(b'uuid'))
return proto
def filter_smudge():
# Operate on stdin/stdout in binary mode
firstline = next(stdin)
if firstline == b"bigstore\n":
hash_function_name = next(stdin)
hexdigest = next(stdin)
source_filename = object_filename(
bytes_to_native_str(hash_function_name)[:-1],
bytes_to_native_str(hexdigest)[:-1])
try:
with open(source_filename):
pass
except IOError:
stdout.write(firstline)
stdout.write(hash_function_name)
stdout.write(hexdigest)
else:
with open(source_filename, 'rb') as file:
for line in file:
stdout.write(line)
else:
stdout.write(firstline)
for line in stdin:
stdout.write(line)
def _write_handle(self, handle, data, response=True):
if response:
return self.requester.write_by_handle(handle,
bytes_to_native_str(data))
else:
return self.requester.write_without_response_by_handle(
handle, bytes_to_native_str(data))
def test_str_encode_decode_with_py2_str_arg(self):
# Try passing a standard Py2 string (as if unicode_literals weren't imported)
b = str(TEST_UNICODE_STR).encode(utils.bytes_to_native_str(b'utf-8'))
self.assertTrue(isinstance(b, bytes))
self.assertFalse(isinstance(b, str))
s = b.decode(utils.bytes_to_native_str(b'utf-8'))
self.assertTrue(isinstance(s, str))
self.assertEqual(s, TEST_UNICODE_STR)
def test_str_encode_decode_with_py2_str_arg(self):
# Try passing a standard Py2 string (as if unicode_literals weren't imported)
b = str(TEST_UNICODE_STR).encode(utils.bytes_to_native_str(b'utf-8'))
self.assertTrue(isinstance(b, bytes))
self.assertFalse(isinstance(b, str))
s = b.decode(utils.bytes_to_native_str(b'utf-8'))
self.assertTrue(isinstance(s, str))
self.assertEqual(s, TEST_UNICODE_STR)
def convert_from_bytes(cls, elem):
if isinstance(elem, bytes):
elem = bytes_to_native_str(elem)
elif isinstance(elem, list):
elem = [cls.convert_from_bytes(e) for e in elem]
elif isinstance(elem, dict):
elem = OrderedDict(
(bytes_to_native_str(n), cls.convert_from_bytes(e))
for n, e in elem.items())
return elem
def convert_from_bytes(cls, elem):
if isinstance(elem, bytes):
elem = bytes_to_native_str(elem)
elif isinstance(elem, list):
elem = [cls.convert_from_bytes(e) for e in elem]
elif isinstance(elem, dict):
elem = OrderedDict(
(bytes_to_native_str(n), cls.convert_from_bytes(e))
for n, e in elem.items())
return elem
def config_gen_csv(self, keys, as_type=None, output=None):
"""
Generate csv file or print it out to STDOUT
"""
if keys is None:
print(
'No --keys specified, please make sure you have at least one key to generate'
)
return
# split --keys=a,b,c,d to list [a,b,c,d]
keys = [c.strip() for c in keys.split(',')]
image_list = None
if as_type == "image" and not self.runtime.rpms:
image_list = self.runtime.image_metas()
if as_type == "rpm" and not self.runtime.images:
image_list = self.runtime.rpm_metas()
if image_list is None:
print(
'Not correct --type specified (--type image or --type rpm). '
'Or not consistent with global options: --images/-i and --rpms/-r'
)
return
def _write_rows(w):
# write header
w.writerow(keys)
# write values
for value in image_list:
value_list = []
for k in keys:
if k == "key":
value_list.append(value.config_filename)
else:
value_list.append(value.config.get(k, None))
w.writerow(value_list)
if output is None:
writer = csv.writer(sys.stdout,
delimiter=bytes_to_native_str(b','),
quotechar=bytes_to_native_str(b'"'),
quoting=csv.QUOTE_MINIMAL)
_write_rows(writer)
return
with io.open(output, mode='w', encoding="utf-8") as csv_file:
writer = csv.writer(csv_file,
delimiter=bytes_to_native_str(b','),
quotechar=bytes_to_native_str(b'"'),
quoting=csv.QUOTE_MINIMAL)
_write_rows(writer)
def encrypt(self, plaintext):
# type: (AnyStr) -> _OlmMessage
"""Encrypts a message using the session. Returns the ciphertext as an
base64 encoded strin on success. Raises OlmSessionError on failure. If
there weren't enough random bytes to encrypt the message the error
message for the exception will be NOT_ENOUGH_RANDOM.
Args:
plaintext(str): The plaintext message that will be encrypted.
"""
byte_plaintext = to_bytes(plaintext)
r_length = lib.olm_encrypt_random_length(self._session)
random = URANDOM(r_length)
random_buffer = ffi.new("char[]", random)
message_type = lib.olm_encrypt_message_type(self._session)
self._check_error(message_type)
ciphertext_length = lib.olm_encrypt_message_length(
self._session, len(plaintext)
)
ciphertext_buffer = ffi.new("char[]", ciphertext_length)
plaintext_buffer = ffi.new("char[]", byte_plaintext)
self._check_error(lib.olm_encrypt(
self._session,
plaintext_buffer, len(byte_plaintext),
random_buffer, r_length,
ciphertext_buffer, ciphertext_length,
))
if message_type == lib.OLM_MESSAGE_TYPE_PRE_KEY:
return OlmPreKeyMessage(
bytes_to_native_str(ffi.unpack(
ciphertext_buffer,
ciphertext_length
)))
elif message_type == lib.OLM_MESSAGE_TYPE_MESSAGE:
return OlmMessage(
bytes_to_native_str(ffi.unpack(
ciphertext_buffer,
ciphertext_length
)))
else: # pragma: no cover
raise ValueError("Unknown message type")
def encrypt(self, plaintext):
# type: (AnyStr) -> _OlmMessage
"""Encrypts a message using the session. Returns the ciphertext as a
base64 encoded string on success. Raises OlmSessionError on failure.
Args:
plaintext(str): The plaintext message that will be encrypted.
"""
byte_plaintext = to_bytearray(plaintext)
r_length = lib.olm_encrypt_random_length(self._session)
random = URANDOM(r_length)
try:
message_type = lib.olm_encrypt_message_type(self._session)
self._check_error(message_type)
ciphertext_length = lib.olm_encrypt_message_length(
self._session, len(byte_plaintext))
ciphertext_buffer = ffi.new("char[]", ciphertext_length)
self._check_error(
lib.olm_encrypt(
self._session,
ffi.from_buffer(byte_plaintext),
len(byte_plaintext),
ffi.from_buffer(random),
r_length,
ciphertext_buffer,
ciphertext_length,
))
finally:
# clear out copies of plaintext
if byte_plaintext is not plaintext:
for i in range(0, len(byte_plaintext)):
byte_plaintext[i] = 0
if message_type == lib.OLM_MESSAGE_TYPE_PRE_KEY:
return OlmPreKeyMessage(
bytes_to_native_str(
ffi.unpack(ciphertext_buffer, ciphertext_length)))
elif message_type == lib.OLM_MESSAGE_TYPE_MESSAGE:
return OlmMessage(
bytes_to_native_str(
ffi.unpack(ciphertext_buffer, ciphertext_length)))
else: # pragma: no cover
raise ValueError("Unknown message type")
def asDataFrame(self, rowIdAndVersionInIndex=True):
test_import_pandas()
import pandas as pd
try:
#Handle bug in pandas 0.19 requiring quotechar to be str not unicode or newstr
quoteChar = bytes_to_native_str(bytes(self.quoteCharacter)) if six.PY2 else self.quoteCharacter
## assign line terminator only if for single character
## line terminators (e.g. not '\r\n') 'cause pandas doesn't
## longer line terminators. See:
## https://github.com/pydata/pandas/issues/3501
## "ValueError: Only length-1 line terminators supported"
df = pd.read_csv(self.filepath,
sep=self.separator,
lineterminator=self.lineEnd if len(self.lineEnd) == 1 else None,
quotechar=quoteChar,
escapechar=self.escapeCharacter,
header = 0 if self.header else None,
skiprows=self.linesToSkip)
except pd.parser.CParserError as ex1:
df = pd.DataFrame()
if rowIdAndVersionInIndex and "ROW_ID" in df.columns and "ROW_VERSION" in df.columns:
## combine row-ids (in index) and row-versions (in column 0) to
## make new row labels consisting of the row id and version
## separated by a dash.
df.index = row_labels_from_id_and_version(zip(df["ROW_ID"], df["ROW_VERSION"]))
del df["ROW_ID"]
del df["ROW_VERSION"]
return df
def callback(self, monitorId, fileList):
"""
Callback required by FSEvents.FSEventStream.
:Parameters:
:return: No explicit return value.
"""
eventList = []
for fileEvent in fileList:
fileId = fileEvent[0]
if isbytes(fileId):
fileId = bytes_to_native_str(fileId)
info = monitors.EventInfo(fileId, fileEvent[1])
eventList.append(info)
proxy = self.proxies[monitorId]
try:
self.log.info('Event notification on monitor id= %s', monitorId)
self.log.debug(' ...notifications are: %s', str(eventList))
proxy.fsEventHappened(native_str(monitorId), eventList)
except Exception as e:
self.log.info('Callback to monitor id=' + monitorId +
' failed. Reason: ' + str(e))
def decrypt(self, message):
# type: (_OlmMessage) -> str
"""Decrypts a message using the session. Returns the plaintext string
on success. Raises OlmSessionError on failure. If the base64 couldn't
be decoded then the error message will be "INVALID_BASE64". If the
message is for an unsupported version of the protocol the error message
will be "BAD_MESSAGE_VERSION". If the message couldn't be decoded then
the error message will be "BAD_MESSAGE_FORMAT". If the MAC on the
message was invalid then the error message will be "BAD_MESSAGE_MAC".
Args:
message(OlmMessage): The Olm message that will be decrypted. It can
be either an OlmPreKeyMessage or an OlmMessage.
"""
if not message.ciphertext:
raise ValueError("Ciphertext can't be empty")
byte_ciphertext = to_bytes(message.ciphertext)
ciphertext_buffer = ffi.new("char[]", byte_ciphertext)
max_plaintext_length = lib.olm_decrypt_max_plaintext_length(
self._session, message.message_type, ciphertext_buffer,
len(byte_ciphertext))
plaintext_buffer = ffi.new("char[]", max_plaintext_length)
ciphertext_buffer = ffi.new("char[]", byte_ciphertext)
plaintext_length = lib.olm_decrypt(self._session, message.message_type,
ciphertext_buffer,
len(byte_ciphertext),
plaintext_buffer,
max_plaintext_length)
self._check_error(plaintext_length)
return bytes_to_native_str(
ffi.unpack(plaintext_buffer, plaintext_length))
def addWatch(self, path, mask):
if not self.isPathWatched(path):
try:
if isbytes(path):
path_obj = pathModule.path(bytes_to_native_str(path))
else:
path_obj = pathModule.path(path)
res = pyinotify.WatchManager.add_watch(self,
path,
mask,
rec=False,
auto_add=False,
quiet=False)
self.watchPaths.update(res)
self.watchParams[path] = copy.copy(
self.watchParams[path_obj.parent])
if self.watchParams[path].getRec():
for d in path_obj.dirs():
self.addWatch(d, mask)
if self.isPathWatched(path):
self.log.info('Watch added on: %s' % path)
else:
self.log.info('Unable to add watch on: %s' % path)
except Exception as e:
self.log.error('Unable to add watch on: %s : %s' % (path, e))
def __init__(self, seed):
# type: (bytes) -> None
"""Create a new signing object.
Args:
seed(bytes): the seed to use as the private key for signing. The
seed must have the same length as the seeds generated by
PkSigning.generate_seed().
"""
if not seed:
raise ValueError("seed can't be empty")
self._buf = ffi.new("char[]", lib.olm_pk_signing_size())
self._pk_signing = lib.olm_pk_signing(self._buf)
track_for_finalization(self, self._pk_signing, _clear_pk_signing)
seed_buffer = ffi.new("char[]", seed)
pubkey_length = lib.olm_pk_signing_public_key_length()
pubkey_buffer = ffi.new("char[]", pubkey_length)
ret = lib.olm_pk_signing_key_from_seed(self._pk_signing, pubkey_buffer,
pubkey_length, seed_buffer,
len(seed))
# zero out copies of the seed
lib.memset(seed_buffer, 0, len(seed))
self._check_error(ret)
self.public_key = bytes_to_native_str(
ffi.unpack(pubkey_buffer, pubkey_length))
def sign(self, message):
# type: (AnyStr) -> str
"""Signs a message with this account.
Signs a message with the private ed25519 identity key of this account.
Returns the signature.
Raises OlmAccountError on failure.
Args:
message(str): The message to sign.
"""
bytes_message = to_bytearray(message)
out_length = lib.olm_account_signature_length(self._account)
out_buffer = ffi.new("char[]", out_length)
try:
self._check_error(
lib.olm_account_sign(self._account,
ffi.from_buffer(bytes_message),
len(bytes_message), out_buffer,
out_length))
finally:
# clear out copies of the message, which may be plaintext
if bytes_message is not message:
for i in range(0, len(bytes_message)):
bytes_message[i] = 0
return bytes_to_native_str(ffi.unpack(out_buffer, out_length))
def assertEqual(self, op_list1, op_list2):
if isinstance(op_list1, list) and isinstance(op_list2, list):
for op in op_list1 + op_list2:
if isinstance(op, caffe2_pb2.OperatorDef):
op.ClearField(bytes_to_native_str(b'uuid'))
return super(TestGradientCalculation, self).assertEqual(
op_list1, op_list2)
def encrypt(self, plaintext):
# type: (AnyStr) -> str
"""Encrypt a message.
Returns the encrypted ciphertext.
Args:
plaintext(str): A string that will be encrypted using the group
session.
"""
byte_plaintext = to_bytes(plaintext)
message_length = lib.olm_group_encrypt_message_length(
self._session, len(byte_plaintext))
message_buffer = ffi.new("char[]", message_length)
plaintext_buffer = ffi.new("char[]", byte_plaintext)
ret = lib.olm_group_encrypt(
self._session,
plaintext_buffer,
len(byte_plaintext),
message_buffer,
message_length,
)
self._check_error(ret)
return bytes_to_native_str(ffi.unpack(message_buffer, message_length))
def __init__(self, uuid):
self._uuid = copy(BLEUUID.BASE_UUID_BYTES)
if isinstance(uuid, UUID):
# Assume that the UUID is correct
self._uuid = bytearray(uuid.bytes)
elif isinstance(uuid, bytes):
self._uuid[2:4] = bytearray(bytes_to_native_str(uuid))
elif isinstance(uuid, str):
if len(uuid) == 4:
# 16-bit UUID
part = int(uuid, 16).to_bytes(2, 'little')
self._uuid[2:4] = bytearray(part)
elif len(uuid) == 8:
# 32-bit UUID
part = int(uuid, 16).to_bytes(4, 'little')
self._uuid[0:4] = bytearray(part)
elif len(uuid) == 36:
# 128-bit UUID
self._uuid = bytearray(UUID(uuid).bytes)
else:
raise ValueError("Invalid UUID")
elif isinstance(uuid, int):
if uuid < 65536:
# 16-bit UUID
part = int(uuid).to_bytes(2, 'little')
self._uuid[2:4] = bytearray(part)
elif uuid < 2**32:
# 32-bit UUID
part = int(uuid).to_bytes(4, 'little')
self._uuid[0:4] = bytearray(part)
else:
raise ValueError("Invalid UUID")
else:
raise ValueError("Invalid UUID (type error)")
def export_session(self, message_index):
# type: (int) -> str
"""Export an inbound group session
Export the base64-encoded ratchet key for this session, at the given
index, in a format which can be used by import_session().
Raises OlmGroupSessionError on failure. The error message for the
exception will be:
* OLM_UNKNOWN_MESSAGE_INDEX if we do not have a session key
corresponding to the given index (ie, it was sent before the
session key was shared with us)
Args:
message_index(int): The message index at which the session should
be exported.
"""
export_length = lib.olm_export_inbound_group_session_length(
self._session)
export_buffer = ffi.new("char[]", export_length)
ret = lib.olm_export_inbound_group_session(self._session,
export_buffer,
export_length,
message_index)
self._check_error(ret)
return bytes_to_native_str(ffi.unpack(export_buffer, export_length))
def cleanup_x509_text(txt):
kts = txt.split(b'\n')
kt = [
b' ' + x for x in kts
if len(x) and not (x.startswith(b'----') and x.endswith(b'----'))
]
return bytes_to_native_str(b' ' + b'\n '.join(kt) + b'\n')
def encrypt(self, plaintext):
# type: (AnyStr) -> str
"""Encrypt a message.
Returns the encrypted ciphertext.
Args:
plaintext(str): A string that will be encrypted using the group
session.
"""
byte_plaintext = to_bytearray(plaintext)
message_length = lib.olm_group_encrypt_message_length(
self._session, len(byte_plaintext))
message_buffer = ffi.new("char[]", message_length)
try:
ret = lib.olm_group_encrypt(
self._session,
ffi.from_buffer(byte_plaintext),
len(byte_plaintext),
message_buffer,
message_length,
)
self._check_error(ret)
finally:
# clear out copies of plaintext
if byte_plaintext is not plaintext:
for i in range(0, len(byte_plaintext)):
byte_plaintext[i] = 0
return bytes_to_native_str(ffi.unpack(message_buffer, message_length))
def login_for_jwt(self):
try:
session = requests.Session()
session.headers.update({
'content-type': 'application/json',
'accept': 'application/json'
})
password_prompt = bytes_to_native_str(b"Password :"******"Udacity Login required.")
email = input('Email :')
password = getpass.getpass(password_prompt)
udacity_login(session, self.root_url, email, password)
except requests.exceptions.HTTPError as e:
if e.response.status_code == 403:
raise ProjectAssistantAuthenticationError(
"Authentication failed")
else:
raise e
try:
r = session.post(self.root_url + '/auth_tokens')
r.raise_for_status()
except:
raise ProjectAssistantAuthenticationError("Authentication failed")
jwt = r.json()['auth_token']
return jwt
def decrypt(self, message):
# type (PkMessage) -> str
ephermal_key = to_bytes(message.ephermal_key)
ephermal_key_size = len(ephermal_key)
mac = to_bytes(message.mac)
mac_length = len(mac)
ciphertext = to_bytes(message.ciphertext)
ciphertext_length = len(ciphertext)
max_plaintext_length = lib.olm_pk_max_plaintext_length(
self._pk_decryption,
ciphertext_length
)
plaintext = ffi.new("char[]", max_plaintext_length)
ret = lib.olm_pk_decrypt(
self._pk_decryption,
ephermal_key, ephermal_key_size,
mac, mac_length,
ciphertext, ciphertext_length,
plaintext, max_plaintext_length)
self._check_error(ret)
unpacked_plaintext = (ffi.unpack(
plaintext,
ret
))
return bytes_to_native_str(unpacked_plaintext)
def calculate_mac(self, message, extra_info):
# type: (str, str) -> str
"""Generate a message authentication code based on the shared secret.
Args:
message (str): The message to produce the authentication code for.
extra_info (str): Extra information to mix in when generating the
MAC
Raises OlmSasError on failure.
"""
byte_message = to_bytes(message)
byte_info = to_bytes(extra_info)
mac_length = lib.olm_sas_mac_length(self._sas)
mac_buffer = ffi.new("char[]", mac_length)
self._check_error(
lib.olm_sas_calculate_mac(
self._sas,
ffi.from_buffer(byte_message),
len(byte_message),
ffi.from_buffer(byte_info),
len(byte_info),
mac_buffer,
mac_length
)
)
return bytes_to_native_str(ffi.unpack(mac_buffer, mac_length))
def calculate_mac_long_kdf(self, message, extra_info):
# type: (str, str) -> str
"""Generate a message authentication code based on the shared secret.
This function should not be used unless compatibility with an older
non-tagged Olm version is required.
Args:
message (str): The message to produce the authentication code for.
extra_info (str): Extra information to mix in when generating the
MAC
Raises OlmSasError on failure.
"""
byte_message = to_bytes(message)
byte_info = to_bytes(extra_info)
mac_length = lib.olm_sas_mac_length(self._sas)
mac_buffer = ffi.new("char[]", mac_length)
self._check_error(
lib.olm_sas_calculate_mac_long_kdf(
self._sas,
ffi.from_buffer(byte_message),
len(byte_message),
ffi.from_buffer(byte_info),
len(byte_info),
mac_buffer,
mac_length
)
)
return bytes_to_native_str(ffi.unpack(mac_buffer, mac_length))
def parse_lines(self, lines):
"""Parse the properties from the given configuration file lines"""
for line in lines:
if isbytes(line):
line = bytes_to_native_str(line)
if line.endswith("\n"):
line = line[:-1]
if line.startswith(STOP):
self.cleanup()
break
if self.is_excluded(line):
self.cleanup()
continue
elif not line.strip():
self.cleanup()
continue
elif line.startswith("#"):
self.append(line)
elif "=" in line and self.curr_a != ESCAPED:
self.detect(line)
elif line.endswith("\\"):
self.cont(line[:-1])
else:
self.cont(line)
self.cleanup() # Handle no newline at end of file
return self.properties
def __init__(self, uuid):
self._uuid = copy(BLEUUID.BASE_UUID_BYTES)
if isinstance(uuid, UUID):
# Assume that the UUID is correct
self._uuid = bytearray(uuid.bytes)
elif isinstance(uuid, bytes):
self._uuid[2:4] = bytearray(bytes_to_native_str(uuid))
elif isinstance(uuid, str):
if len(uuid) == 4:
# 16-bit UUID
part = int(uuid, 16).to_bytes(2, 'little')
self._uuid[2:4] = bytearray(part)
elif len(uuid) == 8:
# 32-bit UUID
part = int(uuid, 16).to_bytes(4, 'little')
self._uuid[0:4] = bytearray(part)
elif len(uuid) == 36:
# 128-bit UUID
self._uuid = bytearray(UUID(uuid).bytes)
else:
raise ValueError("Invalid UUID")
elif isinstance(uuid, int):
if uuid < 65536:
# 16-bit UUID
part = int(uuid).to_bytes(2, 'little')
self._uuid[2:4] = bytearray(part)
elif uuid < 2**32:
# 32-bit UUID
part = int(uuid).to_bytes(4, 'little')
self._uuid[0:4] = bytearray(part)
else:
raise ValueError("Invalid UUID")
else:
raise ValueError("Invalid UUID (type error)")
def from_pickle(cls, pickle, passphrase=""):
# types: (bytes, str) -> PkDecryption
if not pickle:
raise ValueError("Pickle can't be empty")
byte_key = to_bytes(passphrase)
key_buffer = ffi.new("char[]", byte_key)
pickle_buffer = ffi.new("char[]", pickle)
pubkey_length = lib.olm_pk_key_length()
pubkey_buffer = ffi.new("char[]", pubkey_length)
obj = cls.__new__(cls)
ret = lib.olm_unpickle_pk_decryption(
obj._pk_decryption,
key_buffer, len(byte_key),
pickle_buffer, len(pickle),
pubkey_buffer, pubkey_length)
obj._check_error(ret)
obj.public_key = bytes_to_native_str(ffi.unpack(
pubkey_buffer,
pubkey_length
))
return obj
def login_for_jwt(self):
try:
session = requests.Session()
session.headers.update({'content-type':'application/json', 'accept': 'application/json'})
password_prompt = bytes_to_native_str(b"Password :"******"Udacity Login required.")
email = input('Email :')
password = getpass.getpass(password_prompt)
udacity_login(session, self.root_url, email, password)
elif self.id_provider == 'gt':
print("GT Login required.")
username = input('Username :'******'developer':
print("Developer Login required.")
username = input('Username :'******'/auth_tokens')
r.raise_for_status()
jwt = r.json()['auth_token']
return jwt
def encrypt(self, plaintext):
# type: (AnyStr) -> PkMessage
"""Encrypt a message.
Returns the encrypted PkMessage.
Args:
plaintext(str): A string that will be encrypted using the
PkEncryption object.
"""
byte_plaintext = to_bytearray(plaintext)
r_length = lib.olm_pk_encrypt_random_length(self._pk_encryption)
random = URANDOM(r_length)
random_buffer = ffi.new("char[]", random)
ciphertext_length = lib.olm_pk_ciphertext_length(
self._pk_encryption, len(byte_plaintext))
ciphertext = ffi.new("char[]", ciphertext_length)
mac_length = lib.olm_pk_mac_length(self._pk_encryption)
mac = ffi.new("char[]", mac_length)
ephemeral_key_size = lib.olm_pk_key_length()
ephemeral_key = ffi.new("char[]", ephemeral_key_size)
ret = lib.olm_pk_encrypt(self._pk_encryption,
ffi.from_buffer(byte_plaintext),
len(byte_plaintext), ciphertext,
ciphertext_length, mac, mac_length,
ephemeral_key, ephemeral_key_size,
random_buffer, r_length)
try:
self._check_error(ret)
finally: # pragma: no cover
# clear out copies of plaintext
if byte_plaintext is not plaintext:
for i in range(0, len(byte_plaintext)):
byte_plaintext[i] = 0
message = PkMessage(
bytes_to_native_str(ffi.unpack(ephemeral_key, ephemeral_key_size)),
bytes_to_native_str(ffi.unpack(mac, mac_length)),
bytes_to_native_str(ffi.unpack(ciphertext, ciphertext_length)))
return message
def test_bytes_to_native_str(self):
"""
Test for issue #47
"""
b = bytes(b'abc')
s = bytes_to_native_str(b)
if PY2:
self.assertEqual(s, b)
else:
self.assertEqual(s, 'abc')
self.assertTrue(isinstance(s, native_str))
self.assertEqual(type(s), native_str)
def filter_smudge():
# Operate on stdin/stdout in binary mode
firstline = next(stdin)
if firstline == b"bigstore\n":
hash_function_name = next(stdin)
hexdigest = next(stdin)
source_filename = object_filename(bytes_to_native_str(hash_function_name)[:-1],
bytes_to_native_str(hexdigest)[:-1])
try:
with open(source_filename):
pass
except IOError:
stdout.write(firstline)
stdout.write(hash_function_name)
stdout.write(hexdigest)
else:
with open(source_filename, 'rb') as file:
for line in file:
stdout.write(line)
else:
stdout.write(firstline)
for line in stdin:
stdout.write(line)
def _get_package_zip(self, path):
# Build the zip file
zip_bytes = io.BytesIO()
zipf = zipfile.ZipFile(zip_bytes, "w", zipfile.ZIP_DEFLATED)
with cd(path):
for file_to_package in self._get_files_to_package():
zipf.write(file_to_package)
zipf.close()
zipf_processed = self._process_zip_file(zipfile.ZipFile(zip_bytes))
fp = zipf_processed.fp
zipf_processed.close()
return bytes_to_native_str(base64.b64encode(fp.getvalue()))
def application(environ, start_response):
print "app"
# the environment variable CONTENT_LENGTH may be empty or missing
try:
request_body_size = int(environ.get('CONTENT_LENGTH', 0))
except (ValueError):
request_body_size = 0
# When the method is POST the variable will be sent
# in the HTTP request body which is passed by the WSGI server
# in the file like wsgi.input environment variable.
buf = environ['wsgi.input'].read(request_body_size)
global job_queue
updater = Updater("207443777:AAGuMP5nIJMqbFKILRmVuuAz8in7PfiWdjA")
job_queue = updater.job_queue
# Get the dispatcher to register handlers
dp = updater.dispatcher
# on different commands - answer in Telegram
dp.addHandler(CommandHandler("start", start))
dp.addHandler(CommandHandler("help", start))
dp.addHandler(CommandHandler("set", set))
dp.addHandler(InlineQueryHandler(inlinequery))
# log all errors
dp.addErrorHandler(error)
updater.bot.setWebhook("https://afternoon-shelf-83103.herokuapp.com")
# Start the Bot
#updater.start_polling()
json_string = bytes_to_native_str(buf)
update = Update.de_json(json.loads(json_string))
dp.processUpdate(update)
start_response('200 OK', [('Content-Type', 'text/plain')])
return ['']
# Block until the you presses Ctrl-C or the process receives SIGINT,
# SIGTERM or SIGABRT. This should be used most of the time, since
# start_polling() is non-blocking and will stop the bot gracefully.
import time
def do_POST(self):
self.logger.debug("Webhook triggered")
try:
self._validate_post()
clen = self._get_content_len()
except _InvalidPost as e:
self.send_error(e.http_code)
self.end_headers()
else:
buf = self.rfile.read(clen)
json_string = bytes_to_native_str(buf)
self.send_response(200)
self.end_headers()
self.logger.debug("Webhook received data: " + json_string)
update = Update.de_json(json.loads(json_string))
self.logger.info("Received Update with ID %d on Webhook" % update.update_id)
self.server.update_queue.put(update)
# Try and install Skyfield data
try:
from caput import time as ctime
# Force download of data
sf = ctime.SkyfieldWrapper(
ephemeris='http://e-mode.phas.ubc.ca/~jrs65/de421.bsp'
)
sf.reload()
# Set package data to be installed alongside skyfield
skyfield_data = {
# TODO: Py3 remove this hack needed to work around a setuptools bug
bytes_to_native_str(b'caput'): [
'data/Leap_Second.dat',
'data/de421.bsp',
'data/deltat.data',
'data/deltat.preds'
]
}
except:
import warnings
warnings.warn("Could not install additional Skyfield data.")
skyfield_data = {}
setup(
name='caput',
version=__version__,
def write(self, data):
self.ws.send(b64encode(bytes_to_native_str(bytes(data))))
('mf2.json', ['mf2-from-json.html', 'mf2.html'], microformats2.json_to_html,
# we do not format h-media photos properly in html
('note_with_composite_photo',)),
# not ready yet
# ('mf2.html', ['as-from-mf2.json', 'as.json'], html_to_activity, ()),
('as.json', ['feed-from-as.json', 'feed.json'], activity_to_jsonfeed, ()),
('feed.json', ['as-from-feed.json', 'as.json'], jsonfeed_to_activity, ()),
('as.json', ['as2-from-as.json', 'as2.json'], as2.from_as1, ()),
('as2.json', ['as-from-as2.json', 'as.json'], as2.to_as1, ()),
('as.json', ['rss.xml'], rss_from_activities, ()),
)
test_funcs = {}
for src_ext, dst_exts, fn, excludes in mappings:
for src, dst in filepairs(src_ext, dst_exts):
if any(dst.startswith(exclude) for exclude in excludes):
continue
expected = read(dst)
original = read(src)
test_name = (
'test_%s_%s' % (fn.__name__, src[:-len(src_ext)])
).replace('.', '_').replace('-', '_').strip('_')
test_funcs[test_name] = create_test_function(fn, original, expected)
os.chdir(prevdir)
TestDataTest = type(bytes_to_native_str(b'TestDataTest'), (testutil.TestCase,),
test_funcs)
# -*- coding: utf-8 -*-
"""Wrapper for :mod:`array` to simplify and make future compatible.
Not a complete wrapper. New routines added as required.
:copyright: Copyright (c) 2015 Bivio Software, Inc. All Rights Reserved.
:license: http://www.apache.org/licenses/LICENSE-2.0.html
"""
from __future__ import absolute_import, division, print_function
from future.utils import bytes_to_native_str
import array
#: Future-proof typecode for double
DOUBLE_TYPECODE = bytes_to_native_str('d')
#: Future-proof typecode for float
FLOAT_TYPECODE = bytes_to_native_str(b'f')
def new_double(*args, **kwargs):
"""Creates a new double ("d") array
Args are the same as :func:`array.array` except for typecode,
which is passed by this module.
Returns:
array.array: New, initialized array
"""
return array.array(DOUBLE_TYPECODE, *args, **kwargs)
def read_fastfood(filename, return_dtype=None, need_header=None):
if return_dtype is None:
msg = "Return data-type must be set and a valid numpy data-type"
raise ValueError(msg)
import struct
with open(filename, "rb") as f:
skip1 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
idat = struct.unpack(bytes_to_native_str(b'@iiiii'),
f.read(20))[0:5]
skip2 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
assert skip1 == 20 and skip2 == 20,\
"fast-food file seems to be incorrect (reading idat)"
ngal = idat[1]
if need_header is not None:
# now read fdat
skip1 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
fdat = struct.unpack(bytes_to_native_str(b'@fffffffff'),
f.read(36))[0:9]
skip2 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
assert skip1 == 36 and skip2 == 36,\
"fast-food file seems to be incorrect (reading fdat )"
skip1 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
znow = struct.unpack(bytes_to_native_str(b'@f'), f.read(4))[0]
skip2 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
assert skip1 == 4 and skip2 == 4,\
"fast-food file seems to be incorrect (reading redshift)"
else:
fdat_bytes = 4 + 36 + 4
znow_bytes = 4 + 4 + 4
# seek over the fdat + znow fields + padding bytes
# from current position
f.seek(fdat_bytes + znow_bytes, 1)
# read the padding bytes for the x-positions
skip1 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
assert skip1 == ngal * 4 or skip1 == ngal * 8, \
"fast-food file seems to be corrupt (padding bytes)"
# seek back 4 bytes from current position
f.seek(-4, 1)
pos = {}
for field in 'xyz':
skip1 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
assert skip1 == ngal * 4 or skip1 == ngal * 8, \
"fast-food file seems to be corrupt (padding bytes a)"
# the next division must be the integer division
input_dtype = np.float32 if skip1 // ngal == 4 else np.float
array = np.fromfile(f, input_dtype, ngal)
skip2 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
pos[field] = array if dtype is None else dtype(array)
x = pos['x']
y = pos['y']
z = pos['z']
if need_header is not None:
return idat, fdat, znow, x, y, z
else:
return x, y, z
def _read_lengths(self):
"""Reads the lengths from persistent storage, if it does not exist, returns an empty array."""
lengths = array(bytes_to_native_str(b'l'))
if op.isfile(self._lengths_file):
_read_full_file(lengths, self._lengths_file)
return lengths
"""Python 2 / 3 compatibility layer. The functions are copied from http://python-future.org/ to avoid the extra dependency. """ # TODO: for some reason defining the `bytes_to_native_str` function # ourselves doesn't work ... maybe they do some monkey buiseness on import? from future.utils import bytes_to_native_str #def bytes_to_native_str(b, encoding='utf-8'): # return b.decode(encoding) # See http://python-future.org/stdlib_incompatibilities.html#array-array ARRAY_DOUBLE_TYPECODE = bytes_to_native_str(b'd')
def json_rpc_loads(s):
return json.loads(bytes_to_native_str(s), object_hook=_parse_rpc_message)
def e(cls):
warnings.warn("telegram.Emoji is being deprecated, please see https://git.io/v6DeB")
return bytes_to_native_str(b)
def to_str(self, serial_elem, pretty_print=False, # @ReservedAssignment @IgnorePep8
xml_declaration=False, encoding='UTF-8', **kwargs): # @UnusedVariable @IgnorePep8
return bytes_to_native_str(
etree.tostring(serial_elem, encoding=encoding,
pretty_print=pretty_print,
xml_declaration=xml_declaration))
def _write_handle(self, handle, data, response=True):
if response:
return self.requester.write_by_handle(handle, bytes_to_native_str(data))
else:
return self.requester.write_without_response_by_handle(handle, bytes_to_native_str(data))
def test_get_package_zip(self):
with temporary_dir() as path:
# add package.xml
with open(os.path.join(path, "package.xml"), "w") as f:
f.write(
"""<?xml version="1.0" encoding="UTF-8"?>
<Package xmlns="http://soap.sforce.com/2006/04/metadata">
<version>45.0</version>
</Package>"""
)
# add lwc
lwc_path = os.path.join(path, "lwc")
os.mkdir(lwc_path)
# add lwc linting files (not included in zip)
lwc_ignored_files = [".eslintrc.json", "jsconfig.json"]
for lwc_ignored_file in lwc_ignored_files:
touch(os.path.join(lwc_path, lwc_ignored_file))
# add lwc component
lwc_component_path = os.path.join(lwc_path, "myComponent")
os.mkdir(lwc_component_path)
# add lwc component files included in zip (in alphabetical order)
lwc_component_files = [
{"name": "myComponent.html"},
{"name": "myComponent.js"},
{
"name": "myComponent.js-meta.xml",
"body:": """<?xml version="1.0" encoding="UTF-8"?>
<LightningComponentBundle xmlns="http://soap.sforce.com/2006/04/metadata" fqn="myComponent">
<apiVersion>45.0</apiVersion>
<isExposed>false</isExposed>
</LightningComponentBundle>""",
},
{"name": "myComponent.svg"},
{"name": "myComponent.css"},
]
for lwc_component_file in lwc_component_files:
with open(
os.path.join(lwc_component_path, lwc_component_file.get("name")),
"w",
) as f:
if lwc_component_file.get("body") is not None:
f.write(lwc_component_file.get("body"))
# add lwc component files not included in zip
for lwc_ignored_file in lwc_ignored_files:
touch(os.path.join(lwc_component_path, lwc_ignored_file))
# add lwc component sub-directory and files not included in zip
lwc_component_test_path = os.path.join(lwc_component_path, "__tests__")
os.mkdir(lwc_component_test_path)
touch(os.path.join(lwc_component_test_path, "test.js"))
# add classes
classes_path = os.path.join(path, "classes")
os.mkdir(classes_path)
class_files = [
{
"name": "MyClass.cls-meta.xml",
"body": """<?xml version="1.0" encoding="UTF-8"?>
<ApexClass xmlns="http://soap.sforce.com/2006/04/metadata">
<apiVersion>45.0</apiVersion>
<status>Active</status>
</ApexClass>
""",
},
{"name": "MyClass.cls"},
]
for class_file in class_files:
with open(os.path.join(classes_path, class_file.get("name")), "w") as f:
if class_file.get("body") is not None:
f.write(class_file.get("body"))
# add objects
objects_path = os.path.join(path, "objects")
os.mkdir(objects_path)
object_file_names = ["Account.object", "Contact.object", "CustomObject__c"]
object_file_names.sort()
for object_file_name in object_file_names:
touch(os.path.join(objects_path, object_file_name))
# add sub-directory of objects (that doesn't really exist)
objects_sub_path = os.path.join(objects_path, "does-not-exist-in-schema")
os.mkdir(objects_sub_path)
touch(os.path.join(objects_sub_path, "some.file"))
# test
task = create_task(
Deploy,
{
"path": path,
"namespace_tokenize": "ns",
"namespace_inject": "ns",
"namespace_strip": "ns",
},
)
zip_bytes = io.BytesIO()
zipf = zipfile.ZipFile(zip_bytes, "w", zipfile.ZIP_DEFLATED)
with cd(path):
for file_to_package in task._get_files_to_package():
zipf.write(file_to_package)
zipf.close()
zipf_processed = task._process_zip_file(zipfile.ZipFile(zip_bytes))
fp = zipf_processed.fp
zipf_processed.close()
expected = bytes_to_native_str(base64.b64encode(fp.getvalue()))
actual = task._get_package_zip(path)
self.assertEqual(expected, actual)
