def client_write_strings(pipe_name, data_strings):
"""Write strings to a named pipe as a pipe client
Args:
pipe_name (str): name of named pipe
data_strings (list): list of strings to write to pipe
Returns:
bool: True on success, False on failure.
"""
try:
pipe_handle = client_connect_to_pipe(pipe_name)
except pywintypes.error as err:
(winerror, _funcname, _strerror) = err.args
if winerror == 2:
print("Error: No pipe server.")
return False
raise
print("Client Connected to pipe.")
# send filenames to pipe
for data_string in data_strings:
pipe_write(pipe_handle, data_string)
print("Wrote: %s" % data_string)
# sometimes writing these in very fast sequence can make read server
# interpret two data_strings as one message.
# Flush write file buffers to ensure we write this message
win32file.FlushFileBuffers(pipe_handle)
# Close Handle
win32file.CloseHandle(pipe_handle)
return True
def close(self):
if self.view == Pipe.View.SERVER:
if self.transport == Pipe.Transport.SYNCHRONOUS:
w32f.FlushFileBuffers(self.__hPipe)
w32p.DisconnectNamedPipe(self.__hPipe)
else:
self.__hPipe.Close()
w32api.CloseHandle(self.__hPipe)
def superflush(pe_name):
# Flush the file buffers to try to work around a Windows 10 kernel bug,
# https://crbug.com/644525
output_handle = win32file.CreateFile(pe_name, win32file.GENERIC_WRITE,
0, None, win32file.OPEN_EXISTING,
0, 0)
win32file.FlushFileBuffers(output_handle)
output_handle.Close()
def ExecLinkWrapper(self, arch, use_separate_mspdbsrv, *args):
"""Filter diagnostic output from link that looks like:
' Creating library ui.dll.lib and object ui.dll.exp'
This happens when there are exports from the dll or exe.
"""
env = self._GetEnv(arch)
if use_separate_mspdbsrv == 'True':
self._UseSeparateMspdbsrv(env, args)
if sys.platform == 'win32':
args = list(args) # *args is a tuple by default, which is read-only.
args[0] = args[0].replace('/', '\\')
# https://docs.python.org/2/library/subprocess.html:
# "On Unix with shell=True [...] if args is a sequence, the first item
# specifies the command string, and any additional items will be treated as
# additional arguments to the shell itself. That is to say, Popen does the
# equivalent of:
# Popen(['/bin/sh', '-c', args[0], args[1], ...])"
# For that reason, since going through the shell doesn't seem necessary on
# non-Windows don't do that there.
pdb_name = None
pe_name = None
for arg in args:
m = _LINK_PDB_OUT_ARG.match(arg)
if m:
pdb_name = m.group('out')
m = _LINK_EXE_OUT_ARG.match(arg)
if m:
pe_name = m.group('out')
for retry_count in range(_LINKER_RETRIES):
retry = False
link = subprocess.Popen(args, shell=sys.platform == 'win32', env=env,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
# Read output one line at a time as it shows up to avoid OOM failures when
# GBs of output is produced.
for line in link.stdout:
if (not line.startswith(' Creating library ') and
not line.startswith('Generating code') and
not line.startswith('Finished generating code')):
m = _LINK_ERROR.match(line)
if m:
error_code = int(m.groups()[0])
if error_code == _LINKER_RETRY_ERRORS:
print 'Retrying link due to error %d' % error_code
if pdb_name:
shutil.copyfile(pdb_name, pdb_name + 'failure_backup')
retry = True
print line,
result = link.wait()
if not retry:
break
if result == 0 and sys.platform == 'win32':
# Flush the file buffers to try to work around a Windows 10 kernel bug,
# https://crbug.com/644525
output_handle = win32file.CreateFile(pe_name, win32file.GENERIC_WRITE,
0, None, win32file.OPEN_EXISTING, 0, 0)
win32file.FlushFileBuffers(output_handle)
output_handle.Close()
return result
def send(self, msg: Any):
packed_msg = pickle.dumps(msg)
msg_length = struct.pack('>I', len(packed_msg))
# 64k segmentation write
full_msg = msg_length + packed_msg
for i in range(0, len(full_msg), 65536):
win32file.WriteFile(self._pipe,
full_msg[i * 65536:(i + 1) * 65536])
win32file.FlushFileBuffers(self._pipe)
def close(self):
"""Closes the channel to the client.
"""
try:
win32file.WriteFile(self.__pipe_handle, struct.pack("I", 0))
win32file.FlushFileBuffers(self.__pipe_handle)
finally:
win32pipe.DisconnectNamedPipe(self.__pipe_handle)
self.__pipe_handle = None
def write(self, string):
if self._immutable:
raise NotImplemented
if self._redirect_output:
if not string.endswith('\n'):
string += '\n'
win32file.WriteFile(self._stdin_write, string)
win32file.FlushFileBuffers(self._stdin_write)
def close(self):
"""Closes the channel to the client."""
try:
# 2->TODO struct.pack|unpack in python < 2.7.7 does not allow unicode format string.
win32file.WriteFile(self.__pipe_handle,
compat.struct_pack_unicode("I", 0))
win32file.FlushFileBuffers(self.__pipe_handle)
finally:
win32pipe.DisconnectNamedPipe(self.__pipe_handle)
self.__pipe_handle = None
def write(self, d):
ret = 0
try:
if self.pipe_open:
ret = win32file.WriteFile(self.pipe_handle, d)
win32file.FlushFileBuffers(self.pipe_handle)
except:
# try a reconnect
self.pipe_open = False
return ret
def transact(self, string):
try:
# TODO get preferred encoding from application
win32file.WriteFile(self.handle, string.encode('utf-8'))
win32file.FlushFileBuffers(self.handle)
resp = win32file.ReadFile(self.handle, 64 * 1024)
return resp[1].decode('utf-8')
except pywintypes.error as e:
if e.args[0] == winerror.ERROR_BROKEN_PIPE:
raise BrokenPipeError("Broken pipe")
else:
raise BrokenPipeError('Unexpected pipe error: {}'.format(e))
def SendData(self, data):
try:
if not self.hasClient:
win32pipe.ConnectNamedPipe(self.pipe, None)
self.hasClient = True
win32file.WriteFile(self.pipe, data)
win32file.FlushFileBuffers(self.pipe)
except Exception:
if self.hasClient:
win32pipe.DisconnectNamedPipe(self.pipe)
self.hasClient = False
return
def Close(self):
'''
Close out this pipe cleanly, waiting for reader to empty its data.
'''
# Close the pipe.
print('Closing ' + self.pipe_path)
# The routine for closing is described here:
# https://docs.microsoft.com/en-us/windows/win32/ipc/named-pipe-operations
win32file.FlushFileBuffers(self.pipe_file)
win32pipe.DisconnectNamedPipe(self.pipe_file)
win32file.CloseHandle(self.pipe_file)
return
def _write_fifo(self, data):
try:
errCode, _ = win32file.WriteFile(self.of, data)
win32file.FlushFileBuffers(self.of)
if errCode != 0:
logger.error('Error writing data to %s: %d' %
(self.out_fifo, errCode))
except win32api.error, (code, fn, details):
if code == 109 or code == 232:
logger.warn('Remote end stopped reading')
self._not_piped()
self._reset_of()
else:
raise
def flush( self ):
if self.bWindows:
import win32file
try:
#~ print("flush 1")
win32file.FlushFileBuffers( self.handle )
#~ print("flush 2")
#~ os.fsync( win32file._get_osfhandle(self.handle) ) # _get_osfhandle to go from fd from file.open() to system handle not from this win32 handle...
#~ print("flush 3")
pass
except win32file.error as err:
#~ print("WRN: flush: during flush error: %s" % str(err) )
pass
return
self.handle.flush()
os.fsync(self.handle.fileno())
def _writefifo(self, process_info, data: typing.Iterable[str]):
encoded = (("{}:{},".format(len(line), line)).encode('utf8') for line in data)
if _is_linux:
with open(process_info["control"], "wb") as fd:
for line in encoded:
self.logger.debug("_writefifo '%s'", line)
fd.write(line)
elif _is_win:
unique_id = process_info["unique_id"]
handle = self._control_handles[unique_id]
for line in encoded:
self.logger.debug("_writefifo '%s'", line)
errcode, nbytes = win32file.WriteFile(handle, line)
if nbytes != len(line):
raise RuntimeError("Short write {}/{}".format(nbytes, len(line)))
# Block until all data has been read by the client
win32file.FlushFileBuffers(handle)
def start_input_output_listening_loop(self):
keep_looping = True
while (keep_looping):
try:
is_connected = win32pipe.ConnectNamedPipe(
self.pipe_handle, None)
if is_connected is False:
print(is_connected)
else:
data = win32file.ReadFile(self.pipe_handle, 65536)
win32file.WriteFile(self.pipe_handle, b'Nothing')
if data[1].decode('utf-16') == 'Done':
keep_looping = False
np.save(
"C:/Users/louis/Documents/GitHub/DeepLearningMT4Bridge/"
+ self.file_name, np.array(self.list_of_inputs))
np.save(
"C:/Users/louis/Documents/GitHub/DeepLearningMT4Bridge/sec_"
+ self.file_name,
np.array(self.list_of_secondary_inputs))
np.save(
"C:/Users/louis/Documents/GitHub/DeepLearningMT4Bridge/targets_"
+ self.file_name, np.array(self.list_of_targets))
else:
data = str(data[1], 'utf-16').split('|')
self.list_of_secondary_inputs.append(
list(map(np.float32, data[0].split(','))))
self.list_of_targets.append(
list(map(np.float32, data[1].split(','))))
new_input = []
for i in range(2, len(data) - 1):
new_input.append(
list(map(np.float32, data[i].split(','))))
self.list_of_inputs.append(new_input)
win32file.FlushFileBuffers(self.pipe_handle)
win32pipe.DisconnectNamedPipe(self.pipe_handle)
except Exception as e:
print(e)
time.sleep(1000)
def named_pipe_server():
print("Named_Pipe_Server.")
# After 3 clients are connected to it, it will automatically stop providing the service.
for cnt in range(3):
print("Service {} is started.".format(cnt + 1))
# Create named pipe
pipe = win32pipe.CreateNamedPipe(
r'\\.\pipe\ABC', win32pipe.PIPE_ACCESS_DUPLEX,
win32pipe.PIPE_TYPE_MESSAGE | win32pipe.PIPE_READMODE_MESSAGE
| win32pipe.PIPE_WAIT, 1, 65536, 65536, 0, None)
print("Named Pipe is created. Waiting for Client to connect.")
# Enable named pipe and wait for client connection
win32pipe.ConnectNamedPipe(pipe, None)
print("Client is conencted.")
for count in range(10):
# Obtain current date time in predefined format
now = datetime.now()
if cnt % 2 == 0:
dt_string = now.strftime("%d/%m/%Y %H:%M:%S")
else:
dt_string = now.strftime("%d_%m_%Y_%H_%M_%S")
# Encode the string to ASCII
some_data = str.encode(dt_string, encoding="ascii")
# Send the encoded string to client
err, bytes_written = win32file.WriteFile(pipe, some_data)
print(f"Count: {count+1}, Data Sent: {some_data}")
print(
f"WriteFile Return Code: {err}, Number of Bytes Written: {bytes_written}"
)
# Pause for 0.2 second
time.sleep(0.2)
# Ensure all data read by client
win32file.FlushFileBuffers(pipe)
# Disconnect the named pipe
win32pipe.DisconnectNamedPipe(pipe)
# CLose the named pipe
win32file.CloseHandle(pipe)
print(f"Server {cnt+1} is ended.")
print("\n")
def cp(src, dst, blksz, fsync=False):
"""
Copy a file from source to destination.
The destination may be a directory.
Args:
src (str): Source file
dst (str): Destination file or directory
blksz (int): Block size in KB
fsync (bool): Fsync after IO is complete
"""
blksz *= 1024
if os.path.isdir(dst):
dst = os.path.join(dst, os.path.basename(src))
# Open destination file using win32 API
fddst = win32file.CreateFile(dst, win32file.GENERIC_WRITE, 0, None,
win32con.CREATE_ALWAYS, None, None)
try:
# Write file and metadata.
with open(src, 'rb') as fdsrc:
while True:
buf = fdsrc.read(blksz)
if not buf:
break
win32file.WriteFile(fddst, buf)
# Flush and close dst
if fsync:
win32file.FlushFileBuffers(fddst)
except:
raise
finally:
fddst.close()
def flush(self):
win32file.FlushFileBuffers(self.handle)
def close(self):
win32file.FlushFileBuffers(self.pipe)
win32pipe.DisconnectNamedPipe(self.pipe)
win32api.CloseHandle(self.pipe)
def start_prediction_loop(self):
pred = Predictor(filename=self.file_name + "_pred")
data_q = queue.Queue()
for handle_index in range(self.number_of_symbols):
t = threading.Thread(target=self.wait_for_pipe_data,
args=(handle_index, data_q),
daemon=True)
t.start()
keep_looping = True
while keep_looping:
try:
data, symbol_index = data_q.get()
if data[1].decode('utf-16') == 'Done':
pass
# keep_looping = False
else:
self.list_of_inputs = []
self.list_of_secondary_inputs = []
data = str(data[1], 'utf-16').split('|')
# print(data)
self.list_of_secondary_inputs.append(
list(map(np.float32, data[0].split(','))))
new_input = []
for i in range(1, len(data) - 1):
new_input.append(
list(map(np.float32, data[i].split(','))))
self.list_of_inputs.append(new_input)
np.save(
"C:/Users/louis/Documents/GitHub/DeepLearningMT4Bridge/"
+ self.file_name + "_pred",
np.array(self.list_of_inputs))
np.save(
"C:/Users/louis/Documents/GitHub/DeepLearningMT4Bridge/sec_"
+ self.file_name + "_pred",
np.array(self.list_of_secondary_inputs))
prediction = pred.predict()
if (np.argmax(prediction[0][0]) == 2) & (np.argmax(
prediction[1][0]) == 2) & (np.argmax(
prediction[2][0]) == 2) & (np.argmax(
prediction[3][0]) == 2) & (np.argmax(
prediction[4][0]) == 2):
win32file.WriteFile(self.pipe_handles[symbol_index],
b'Buy')
elif (np.argmax(prediction[0][0]) == 1) & (np.argmax(
prediction[1][0]) == 1) & (np.argmax(
prediction[2][0]) == 1) & (np.argmax(
prediction[3][0]) == 1) & (np.argmax(
prediction[4][0]) == 1):
win32file.WriteFile(self.pipe_handles[symbol_index],
b'Sell')
else:
win32file.WriteFile(self.pipe_handles[symbol_index],
b'Pass')
win32file.FlushFileBuffers(self.pipe_handles[symbol_index])
win32pipe.DisconnectNamedPipe(self.pipe_handles[symbol_index])
t = threading.Thread(target=self.wait_for_pipe_data,
args=(symbol_index, data_q),
daemon=True)
t.start()
except Exception as e:
print(e)
time.sleep(1000)
def flush(self):
if not self.is_closed:
win32file.FlushFileBuffers(self.handle)
def flush(self):
if self.is_open():
win32file.FlushFileBuffers(self._handle)
def named_pipe_server(baidu_translate):
while True:
print("Named_Pipe_Server starting.")
try:
# Create named pipe
pipe = win32pipe.CreateNamedPipe(
r'\\.\pipe\BAIDU_TRANSLATE', win32pipe.PIPE_ACCESS_DUPLEX,
win32pipe.PIPE_TYPE_MESSAGE | win32pipe.PIPE_READMODE_MESSAGE
| win32pipe.PIPE_WAIT, win32pipe.PIPE_UNLIMITED_INSTANCES,
65536, 65536, 0, None)
print("Named Pipe is created. Waiting for Client to connect.")
# Enable named pipe and wait for client connection
win32pipe.ConnectNamedPipe(pipe, None)
print("Client is connected.")
cont = True
count = 0
phase = fromLang = toLang = None
while cont:
# Accept std::wstring or TCHAR
resp = win32file.ReadFile(pipe, 65536)
encoding = None
if resp[1][1] != 0:
encoding = 'utf-8'
else:
encoding = 'utf-16LE' # No BOM
msg = resp[1].decode(encoding)
info = re.findall(r"^\[Message #\d+,to=(.*),from=(.*)\](.+)$",
msg)
toLang = info[0][0]
fromLang = info[0][1]
phase = urllib.parse.unquote(info[0][2])
print(f"Phase: {phase}")
try:
if phase and "×" in phase:
phase = phase.replace("×", "ン")
if phase:
data = baidu_translate.translate(phase,
input_lang=fromLang,
output_lang=toLang)
except Exception as e:
raise e
print(f"Translated: {data}")
try:
data = data.encode(encoding)
except:
data = "failed".encode(encoding)
err, bytes_written = win32file.WriteFile(pipe, data)
count += 1
print(
f"WriteFile Return Code: {err}, Number of Bytes Written: {bytes_written}"
)
if err:
cont = False
except pywintypes.error as e:
print(e)
finally:
print("Server exiting")
if pipe:
# Ensure all data read by client
win32file.FlushFileBuffers(pipe)
# Disconnect the named pipe
win32pipe.DisconnectNamedPipe(pipe)
# CLose the named pipe
win32file.CloseHandle(pipe)
print(f"Server ended.")
def flush(self):
import win32file as w
w.FlushFileBuffers(self._handle)
def stop(self):
'''Stop the handler'''
if win32file.FlushFileBuffers(self.pipe):
if win32pipe.DisconnectNamedPipe(self.pipe):
win32api.CloseHandle(self.pipe)
def Write(self, message):
win32file.WriteFile(self._fileHandle, message.encode('utf-8'))
win32file.FlushFileBuffers(self._fileHandle)
return
def Runtime_Test(pure_python=False):
'''
Fleshing out this test to be more thorough.
The interface will use a memory style protocol, where
* pure_python
- Bool, if True then the test is done purely in python, with a local
client accessing the pipe.
- Default is False, leaving the pipe open for x4 to connect to.
'''
# Can either have x4 open it and python listen in, or python open
# it and x4 listen. Or have either open it? Unclear on how these work.
# At any rate, try to open it.
pipe = win32pipe.CreateNamedPipe(
# Note: for some dumb reason, this doesn't use keyword args,
# so arg names included in comments.
# pipeName
pipe_name,
# The lua winapi opens pipes as read/write; try to match that.
# openMode
win32pipe.PIPE_ACCESS_DUPLEX,
# pipeMode
# Set writes to message, reads to message.
# This means reading from the pipe grabs a complete message
# as written, instead of a lump of bytes.
win32pipe.PIPE_TYPE_MESSAGE | win32pipe.PIPE_READMODE_MESSAGE
| win32pipe.PIPE_WAIT,
# nMaxInstances
1,
# nOutBufferSize
65536,
# nInBufferSize
# Can limit this to choke writes and see what errors they give.
# In testing, this needs to be large enough for any single message,
# else the client write fails with no error code (eg. code 0).
# In testing, a closed server and a full pipe generate the same
# error code, so x4 stalling on full buffers will not be supported.
# This buffer should be sized large enough to never fill up.
65536,
# nDefaultTimeOut
300,
# sa
None)
print('Started serving: ' + pipe_name)
# For python testing, kick off a client thread.
if pure_python:
# Set up the reader in another thread.
reader_thread = threading.Thread(target=Pipe_Client_Test)
reader_thread.start()
# For safety, on errors shut down the pipe safely.
try:
# Set up connections.
# This appears to be a stall op that waits for a client to connect.
# Returns 0, an integer for okayish errors (io pending, or pipe already
# connected), or raises an exception on other errors.
# If the client connected first, don't consider that an error, so
# just ignore any error code but let exceptions get raised.
win32pipe.ConnectNamedPipe(pipe, None)
print('Connected to client')
# X4 will write to its output, read from its input.
# These will be read/write transactions to a data table, stored here.
# Loop ends on getting a 'close' command.
data_store = {}
close_requested = False
while not close_requested:
# Get the next control message.
# If the pipe client is closed, this will error.
error, data = win32file.ReadFile(pipe, 64 * 1024)
message = data.decode()
print('Received: ' + message)
# Testing: delay on processing write.
# Used originally to let multiple x4 writes queue up, potentially
# hitting a full buffer.
if 0:
print('Pausing for a moment...')
time.sleep(0.5)
# Handle based on prefix, write or read.
if message.startswith('write:'):
# Expected format is:
# write:[key]value
# (Or possibly a chain of keys? just one for now)
key, value = message.split(':')[1].split(']')
# Pull out the starting bracket.
key = key[1:]
# Save.
data_store[key] = value
elif message.startswith('read:'):
# Expected format is:
# read:[key]
key = message.split(':')[1]
key = key[1:-1]
if key not in data_store:
response = 'error: {} not found'.format(key)
else:
response = data_store[key]
# Pipe the response back.
# Note: data is binary in the pipe, but treated as string in lua,
# plua lua apparently has no integers (just doubles), but does have
# string pack/unpack functions.
# Anyway, it is easiest to make strings canonical for this, for now.
# TODO: consider passing some sort of format specifier so the
# data can be recast in the lua.
# Optionally do a read timeout test, which doesn't return data.
timeout_test = 0
if timeout_test:
print('Suppressing read return; testing timeout.')
else:
error2, bytes_written = win32file.WriteFile(
pipe,
str(response).encode('utf-8'))
print('Returned: ' + response)
elif message == 'close':
# Close the pipe/server when requested.
close_requested = True
else:
print('Unexpected message type')
finally:
# Close the pipe.
print('Closing pipe...')
# The routine for closing is described here:
# https://docs.microsoft.com/en-us/windows/win32/ipc/named-pipe-operations
win32file.FlushFileBuffers(pipe)
win32pipe.DisconnectNamedPipe(pipe)
win32file.CloseHandle(pipe)
return
