def save(self, file, filename):
with open(os.path.join(self.db, filename), "wb") as f:
f.write(serpent.tobytes(file))
for server_name in self.proxy.servers:
if server_name != self.name:
with Pyro4.Proxy("PYRONAME:" + server_name) as storage:
try:
storage._pyroBind()
storage.save(serpent.tobytes(file), filename)
except:
print("Err: Objeto não encontrado...")
def get_autenticacao(self, login_request):
loginAceito = False
login_request1 = serpent.tobytes(login_request)
global thread
thread += 1
thread_metodo = thread
print('Thread', thread_metodo, 'iniciada para autenticação.')
for i in range(len(senhas)):
#print(i)
#print(login_request1)
if (login_request1 == senhas[i]):
loginAceito = True
login_atual.append(login_request1)
if (loginAceito):
resultado = "login aceito"
else:
resultado = "login recusado"
print('Thread', thread_metodo, 'finalizada.')
thread -= 1
return resultado
def read_file(self, filename):
status, recv_filename, size, content = self.remote.read_file(filename)
content = serpent.tobytes(content)
if status:
print("Opening file", filename, "("+str(size)+") :")
print(content.decode("utf-8"))
else:
print(recv_filename)
def execute(self):
banco = Pyro4.Proxy(
"PYRONAME:banco") # use name server object lookup uri shortcut
data = dumps(self.operacao)
resp = loads(tobytes(banco.realizar_operacao(data)))
return resp
def upload_file(self, username, file, data):
#upload a file from client to server in user folder.
outfile = os.path.join(username, file)
# print(data)
data = serpent.tobytes(data)
# print(data)
with open(outfile, "wb") as f:
f.write(data)
def cmdUpload(self, file, data):
raw = serpent.tobytes(data)
print("cmdUpload file \"%s\", size %u bytes" % (file, len(raw)))
# print("raw=%s" % raw)
#upload files
new_file = open(file, "wb")
new_file.write(raw)
new_file.close()
return "Success"
def deserialize_version_struct(classname, d):
assert (d["__class__"] == "VersionStruct")
vs = VersionStruct(deserialize_principal(d["principal"]))
for p, ihandle in d["ihandles"]:
vs.ihandles[deserialize_principal(p)] = ihandle
for p, version_no in d["versions"]:
vs.versions[deserialize_principal(p)] = version_no
assert (isinstance(d["signature"], dict))
vs.signature = serpent.tobytes(d["signature"])
return vs
def send_text(self):
tts_server = Pyro4.Proxy("PYRO:ttsserver@localhost:3000")
audio = tts_server.speak(self.text.get())
pygame.init()
arq = open("teste.mp3", "ab")
for i in audio:
b = serpent.tobytes(i)
arq.write(b)
print(b)
arq.close()
def realizar_operacao(data):
operacao = loads(tobytes(data))
resp = {}
try:
resp['status'] = True
resp['msg'] = Banco().realizar_operacao(operacao)
except SaldoException as e:
resp['status'] = False
resp['msg'] = e.message
return dumps(resp)
def dwld():
fileName = input("Valid file name (inc. extension): ")
files = frontLink.list()
if fileName not in files:
print("File does not exist")
else:
file = frontLink.download(fileName)
fileback = serpent.tobytes(file)
f = open(fileName, 'wb')
f.write(fileback)
f.close()
print("Downloaded: " + fileName)
def testProxyAnnotations(self):
class AnnotationsProxy(Pyro4.core.Proxy):
def _pyroAnnotations(self):
return {"QWER": b"data via old api"}
with AnnotationsProxy(self.objectUri) as p:
Pyro4.core.current_context.annotations = {"XYZZ": b"invalid test data"}
try:
p.response_annotation()
self.fail("should fail")
except ValueError:
pass
Pyro4.core.current_context.annotations = {"XYZZ": b"data from proxy via new api"}
response = p.response_annotation()
self.assertEqual(b"daemon annotation via new api", Pyro4.core.current_context.response_annotations["ANN2"])
# check that the daemon received both the old and the new annotation api data:
daemon_annotations = response["annotations_in_daemon"]
if sys.version_info < (3, 0) and sys.platform != "cli":
self.assertEqual("data from proxy via new api", daemon_annotations["XYZZ"])
self.assertEqual("data via old api", daemon_annotations["QWER"])
else:
self.assertEqual(b"data from proxy via new api", serpent.tobytes(daemon_annotations["XYZZ"]))
self.assertEqual(b"data via old api", serpent.tobytes(daemon_annotations["QWER"]))
def do_test_chunks():
with Pyro4.core.Proxy(uri) as p:
totalsize = 0
begin = time.time()
for chunk in p.download_chunks(datasize*10):
chunk = serpent.tobytes(chunk) # in case of serpent encoded bytes
totalsize += len(chunk)
print(".", end="", flush=True)
assert totalsize == datasize*10
duration = time.time() - begin
totalsize = float(totalsize)
print("It took %.2f seconds to transfer %d mb." % (duration, totalsize / 1024 / 1024))
print("That is %.0f kb/sec. = %.1f mb/sec. (serializer: %s)" % (totalsize / 1024 / duration, totalsize / 1024 / 1024 / duration, Pyro4.config.SERIALIZER))
def testProxyAnnotations(self):
class AnnotationsProxy(Pyro4.core.Proxy):
def _pyroAnnotations(self):
return {"QWER": b"data via old api"}
with AnnotationsProxy(self.objectUri) as p:
Pyro4.core.current_context.annotations = {"XYZZ": b"invalid test data"}
try:
p.response_annotation()
self.fail("should fail")
except ValueError:
pass
Pyro4.core.current_context.annotations = {"XYZZ": b"data from proxy via new api"}
response = p.response_annotation()
self.assertEqual(b"daemon annotation via new api", Pyro4.core.current_context.response_annotations["ANN2"])
# check that the daemon received both the old and the new annotation api data:
daemon_annotations = response["annotations_in_daemon"]
if sys.version_info < (3, 0) and sys.platform != "cli":
self.assertEqual("data from proxy via new api", daemon_annotations["XYZZ"])
self.assertEqual("data via old api", daemon_annotations["QWER"])
else:
self.assertEqual(b"data from proxy via new api", serpent.tobytes(daemon_annotations["XYZZ"]))
self.assertEqual(b"data via old api", serpent.tobytes(daemon_annotations["QWER"]))
def regular_pyro(uri):
blobsize = 10*1024*1024
num_blobs = 10
total_size = 0
start = time.time()
name = threading.currentThread().name
with Pyro4.core.Proxy(uri) as p:
for _ in range(num_blobs):
print("thread {0} getting a blob using regular Pyro call...".format(name))
data = p.get_with_pyro(blobsize)
data = serpent.tobytes(data) # in case of serpent encoded bytes
total_size += len(data)
assert total_size == blobsize*num_blobs
duration = time.time() - start
print("thread {0} done, {1:.2f} Mb/sec.".format(name, total_size/1024.0/1024.0/duration))
def via_iterator(uri):
blobsize = 10*1024*1024
num_blobs = 10
total_size = 0
start = time.time()
name = threading.currentThread().name
with Pyro4.core.Proxy(uri) as p:
for _ in range(num_blobs):
print("thread {0} getting a blob using remote iterators...".format(name))
for chunk in p.iterator(blobsize):
chunk = serpent.tobytes(chunk) # in case of serpent encoded bytes
total_size += len(chunk)
assert total_size == blobsize*num_blobs
duration = time.time() - start
print("thread {0} done, {1:.2f} Mb/sec.".format(name, total_size/1024.0/1024.0/duration))
def runTimeSeriesImageDataUploadOperation(self, timeSeriesImageData):
print(getDT(), "| REMOTE | Has images to upload...")
for timeSeriesImageDatum in timeSeriesImageData:
print(getDT(), "| REMOTE | Uploading datum...")
# DEVELOPERS NOTE:
# - READ: https://pythonhosted.org/Pyro4/tipstricks.html?highlight=image#binarytransfer
# - In essence we need to use the `serpent` library to deserialize and convert to binary.
image_info = timeSeriesImageDatum.get('value')
image_data = serpent.tobytes(
image_info) # in case of serpent encoded bytes
# Perform our submission of our time-series datum into
# the remote API web-service.
msg, wasSuccessful = self.__mikaponicsAPI.postTimeSeriesImageDatum(
self.__token, timeSeriesImageDatum.get('instrument_uuid'),
image_data, timeSeriesImageDatum.get('timestamp'),
timeSeriesImageDatum.get('timestep'))
# CASE 1 OF 2:
# CHECK TO SEE IF THE POST WAS SUCCESSFUL TO OUR REMOTE API
# WEB-SERVICE AND IF SO THEN DO THE FOLLOWING CODE.
if wasSuccessful:
print(getDT(), "| REMOTE | Deleting local image datum...")
self.__storage.deleteTimeSeriesImageDatum(
timeSeriesImageDatum.get('id'))
print(getDT(), "| REMOTE | Deleted local image datum.")
# CASE 2 OF 2:
# ELSE IT WAS FOUND OUT THAT THE POST WAS A FAILURE SO WE WILL
# NEED TO THE FOLLOWING BLOCK OF CODE.
else:
# If there is a message, whatever it is, we will make our
# application just get a new API token to resolve this
# issue.
if msg:
if "You do not have permission to access this API-endpoint" in msg:
print(getDT(), "| REMOTE | Will retry again.")
elif "Token needs to refreshed" in msg:
print(getDT(),
"| REMOTE | Will re-authenticate again...")
self.__token = None
self.runFetchTokenOperationLoop()
# Terminate the loop because we have an error and we do not
# want to iterate through all the data again.
print(getDT(), "| REMOTE | Terminating loop...")
break
def do_test_chunks():
with Pyro4.core.Proxy(uri) as p:
totalsize = 0
begin = time.time()
for chunk in p.download_chunks(datasize * 10):
chunk = serpent.tobytes(chunk) # in case of serpent encoded bytes
totalsize += len(chunk)
print(".", end="", flush=True)
assert totalsize == datasize * 10
duration = time.time() - begin
totalsize = float(totalsize)
print("It took %.2f seconds to transfer %d mb." %
(duration, totalsize / 1024 / 1024))
print("That is %.0f kb/sec. = %.1f mb/sec. (serializer: %s)" %
(totalsize / 1024 / duration, totalsize / 1024 / 1024 / duration,
Pyro4.config.SERIALIZER))
def run(self, graph_def, cluster_dict, job_name, task_index):
graph_def = tobytes(
graph_def) # for remote calls, to undo the serpent deserializer
full_graph_def = meta_graph_pb2.MetaGraphDef.FromString(graph_def)
with self._lock: # because this could be reentrant
self.log.info(
"Attempting to start job {name}, task index {ti}".format(
name=job_name, ti=task_index))
if self.session is not None:
self.log.warning(
"Attempting to assign new slave session when one already exists! Stopping old one"
)
self.stop_and_reset()
# Note: outdir will be absolute because of the argument parser
if len(job_name) == 0:
outdir = self.args.output_directory / "worker.{}".format(
task_index)
else:
outdir = self.args.output_directory / "{n}.{i}".format(
n=job_name, i=task_index)
outdir = outdir.absolute()
if self.args.record_stats:
if outdir.exists():
shutil.rmtree(str(outdir))
outdir.mkdir()
self.session = Session(
graph_def=full_graph_def,
cluster_dict=cluster_dict,
job_name=job_name,
task_index=task_index,
pyro_host=self.args.pyro_ns_host,
pyro_port=self.args.pyro_ns_port,
record_directory=outdir if self.args.record_stats else None)
try:
self.session.run(run_sleep_interval=self.run_sleep_interval,
job_name=job_name,
task_index=task_index,
startup_sleep=self.args.startup_sleep)
except:
self.session = None # failed to launch
raise
else:
if self.args.record_stats:
self.outdir = outdir
def get(self, *args, **kwargs):
"""
Get any parameter being monitored (`monitem`) in NMC control script.
Args:
args: tuple with parameter names to get
kwargs: keyword arguments, usually none
"""
params = []
for arg in args:
if type(arg) == dict:
# recover from Pyro5 serialization if necessary
params.append(serpent.tobytes(arg))
else:
params.append(arg)
self.logger.debug("get(%s): params = %s", logtime(), params)
self.logger.debug("get(%s): kwargs = %s", logtime(), kwargs)
if not self.server_initialized:
self.logger.debug("get({}): initializing".format(logtime()))
self.get_params()
recv_val = kwargs.get("recv_val", 1024)
return_vals = {}
# default values
return_vals = {params[i]: None for i in range(len(params))}
self.logger.debug("get(%s): return_vals = %s", logtime(), return_vals)
if not self._simulated:
self.logger.debug("get: params: {}".format(params))
# at this point ``params`` is byte objects, but we need to strings
# for manipulation
if self.sock is not None:
cmd = "PARAM {}\n".format(" ".join(params))
self.logger.debug("get: sending cmd: {}".format(cmd))
resp = self.command(cmd, recv_val=recv_val)
self.logger.debug("get: resp: {}".format(resp))
resp_list = resp.split(",")
if len(resp_list) == len(params):
return_vals = {
params[i]: resp_list[i].strip()
for i in range(len(params))
}
else:
self.logger.error((
"Discrepency between number of parameters requested and number returned: "
""))
else:
return {"AzimuthAngle": 60, "ElevationAngle": 45}
return return_vals
def _sync(self):
with Pyro4.Proxy("PYRONAME:storage.server.primary") as primary:
primary_files = set(primary.list()['content'])
my_files = set(self.list()['content'])
sync_files = primary_files - my_files
for filename in sync_files:
response = primary.retrieve(filename)
self.save(serpent.tobytes(response['content']), filename)
delete_files = my_files - primary_files
for filename in delete_files:
os.remove(os.path.join(self.db, filename))
def regular_pyro(uri):
blobsize = 10 * 1024 * 1024
num_blobs = 10
total_size = 0
start = time.time()
name = threading.currentThread().name
with Proxy(uri) as p:
for _ in range(num_blobs):
print(
"thread {0} getting a blob using regular Pyro call...".format(
name))
data = p.get_with_pyro(blobsize)
data = serpent.tobytes(data) # in case of serpent encoded bytes
total_size += len(data)
assert total_size == blobsize * num_blobs
duration = time.time() - start
print("thread {0} done, {1:.2f} Mb/sec.".format(
name, total_size / 1024.0 / 1024.0 / duration))
def via_iterator(uri):
blobsize = 10 * 1024 * 1024
num_blobs = 10
total_size = 0
start = time.time()
name = threading.currentThread().name
with Proxy(uri) as p:
for _ in range(num_blobs):
print("thread {0} getting a blob using remote iterators...".format(
name))
for chunk in p.iterator(blobsize):
chunk = serpent.tobytes(
chunk) # in case of serpent encoded bytes
total_size += len(chunk)
assert total_size == blobsize * num_blobs
duration = time.time() - start
print("thread {0} done, {1:.2f} Mb/sec.".format(
name, total_size / 1024.0 / 1024.0 / duration))
def clientUpload(self, fileName, toAll, data):
raw = serpent.tobytes(data)
print("clientUpload file \"%s\", HR=%u, size %u bytes" % (fileName, toAll, len(raw)))
if toAll:
for server in remoteServerList:
server.rmi.cmdUpload(fileName, raw)
else:
small_number_of_stored_files = 100000000
for server in remoteServerList:
if server.no_files < small_number_of_stored_files:
small_number_of_stored_files = server.no_files
for server in remoteServerList:
if server.no_files == small_number_of_stored_files:
server.rmi.cmdUpload(fileName, raw)
break
return "Success transferred %u bytes " % len(raw)
def cmdDownload(self):
download = input("Enter file you would like to download > ").strip()
#checks if file exists locally
if os.path.isfile(download):
print("Error file already exits locally")
return
file_exists = frontEnd.clientDownloadChk(download)
if file_exists != 1:
print("Error file does not exist remotely")
return
data = frontEnd.clientDownload(download)
if data == "":
print("Error file does not exist remotely")
return
#downloads file if doesn't appear locally
tic = time.clock()
converted_data = serpent.tobytes(data)
new_file = open(download, "wb")
new_file.write(converted_data)
new_file.close()
toc = time.clock()
t_time = toc - tic
print("%u bytes transferred in %s seconds" % (len(converted_data), round(t_time,2)))
def update_file(self, filename: str, content: bytes,
size: int) -> (bool, str, int):
content = serpent.tobytes(content)
try:
# pseudo-COW
temp_filename = ".tempfile-" + str(randint(420, 42069))
with open(FileManager.DIRECTORY_PATH + "/" + temp_filename,
"wb") as temp_fd:
write_size = temp_fd.write(content)
# check if write size doesn't match
if write_size != size:
# delete tempfile
self.delete_file(FileManager.DIRECTORY_PATH + "/" +
temp_filename)
return False, "Error on writing. Original write size "\
+ str(size) + " written write size " + str(write_size), -1
# delete original file
self.delete_file(FileManager.DIRECTORY_PATH + "/" + filename)
# rename tempfile to filename
os.rename(FileManager.DIRECTORY_PATH + "/" + temp_filename,
FileManager.DIRECTORY_PATH + "/" + filename)
return True, "File " + filename + " updated", write_size
except FileNotFoundError:
return False, "File " + filename + " could not be found", -1
def recreate_classes(self, literal):
if isinstance(literal, dict) and "data" in literal and literal.get(
"encoding") == "base64":
return serpent.tobytes(literal)
return super().recreate_classes(literal)
def transfer(self, data):
if Pyro4.config.SERIALIZER == "serpent" and type(data) is dict:
data = serpent.tobytes(data) # in case of serpent encoded bytes
print("received %d bytes" % len(data))
return len(data)
def save(self, file, filename):
with open(os.path.join(self.db, filename), "wb") as f:
f.write(serpent.tobytes(file))
def download_client(dropbox, username, filename): #code for local processing download file_content = dropbox.download_file(username, filename) data = serpent.tobytes(file_content) with open(filename, "wb") as f: f.write(data)
def main():
storage = Pyro4.Proxy("PYRONAME:storage.proxy")
running = True
while running:
commands = """
Bem vindo!
Digite a opção desejada:
1 - Criar e enviar arquivo txt
2 - Enviar arquivo pdf
3 - Deletar arquivo
4 - Ler arquivo
5 - Listar arquivos
6 - Sair
"""
print(commands)
opt = int(input("\tInput: "))
if opt == 1:
filename = input("\n\tDigite o nome do arquivo: ")
file = input("\tDigite o conteudo do arquivo: ")
storage.save(bytes(file, 'utf8'), filename)
print("\n\tArquivo enviado. ")
elif opt == 2:
filename = input("\n\tDigite o nome do arquivo: ")
with open(filename, 'rb') as f:
data = f.read()
storage.save(data, filename)
print("\n\tArquivo enviando. ")
elif opt == 3:
filename = input("\n\tDigite o nome do arquivo: ")
print("\n\tResposta: {}".format(storage.delete(filename)))
elif opt == 4:
filename = input("\tDigite o nome do arquivo: ")
response = storage.retrieve(filename)
if response['code'] == '200':
with open(filename, 'wb') as f:
f.write(serpent.tobytes(response['content']))
print("\n\tO arquivo foi salvo.")
else:
print("\n\tO arquivo não existe.")
elif opt == 5:
arquivos = storage.list()['content']
print('\n\tArquivos: ')
for arquivo in arquivos:
print('\t\t{}'.format(arquivo))
elif opt == 6:
running = False
else:
print("\tDigite uma opção válida")
def insertTimeSeriesImageDatum(self, instrument_uuid, image_info, timestamp, timestep):
# DEVELOPERS NOTE:
# - READ: https://pythonhosted.org/Pyro4/tipstricks.html?highlight=image#binarytransfer
# - In essence we need to use the `serpent` library to deserialize and convert to binary.
image_data = serpent.tobytes(image_info) # in case of serpent encoded bytes
return self.__storage.insertTimeSeriesImageDatum(instrument_uuid, image_data, timestamp, timestep)
def upload(self, fileName, file):
fileWrite = serpent.tobytes(file)
newFile = open("files/" + fileName, 'wb')
newFile.write(fileWrite)
newFile.close()
return True
def test_tobytes(self):
obj = b"test"
self.assertIs(obj, serpent.tobytes(obj))
obj = memoryview(b"test")
self.assertIs(obj, serpent.tobytes(obj))
obj = bytearray(b"test")
self.assertIs(obj, serpent.tobytes(obj))
if hasattr(types, "BufferType"):
obj = buffer(b"test")
self.assertIs(obj, serpent.tobytes(obj))
ser = {'data': 'dGVzdA==', 'encoding': 'base64'}
out = serpent.tobytes(ser)
self.assertEqual(b"test", out)
if sys.platform == 'cli':
self.assertIsInstance(
out, str) # ironpython base64 decodes into str type....
else:
self.assertIsInstance(out, bytes)
with self.assertRaises(TypeError):
serpent.tobytes({'@@@data': 'dGVzdA==', 'encoding': 'base64'})
with self.assertRaises(TypeError):
serpent.tobytes({'data': 'dGVzdA==', '@@@encoding': 'base64'})
with self.assertRaises(TypeError):
serpent.tobytes({'data': 'dGVzdA==', 'encoding': 'base99'})
with self.assertRaises(TypeError):
serpent.tobytes({})
with self.assertRaises(TypeError):
serpent.tobytes(42)
def transfer(self, data):
if Pyro5.config.SERIALIZER == "serpent" and type(data) is dict:
data = serpent.tobytes(data) # in case of serpent encoded bytes
print("received %d bytes" % len(data))
return len(data)
# SOCK_DGRAM means a UDP socket
if protocol == 'tc':
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
# Connect to server and send data
sock.connect((HOST, PORT))
sock.sendall(bytes(command + "\n", "utf-8"))
# Receive data from the server
received = str(sock.recv(65100), "utf-8")
finally:
sock.close()
elif protocol == 'rmic':
uri = 'PYRO:node.request@' + HOST + ':' + str(PORT)
warehouse = Pyro4.Proxy(uri)
warehouse.put_values(command, './temp.json')
msg = warehouse.operation()
received = serpent.tobytes(msg).decode('utf-8')
else:
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
# Connect to server and send data
sock.sendto(bytes(command + "\n", "utf-8"), (HOST, PORT))
# Receive data from the server and shut down
received = str(sock.recv(65100), "utf-8")
finally:
sock.close()
#print("Sent: {}".format(command))
print(received)
