def worker(self, timeInterval, keepWorking):
# keeps looking at the queue and empties it to the database
# keepWorking is an event set in the main thread to allow stoping
while keepWorking.is_set():
self.datafile = path + 'DavisVP2_' + datetime.datetime.now().strftime('%m-%d-%y') + '.xlsx'
self.logfile = path + 'status_' + datetime.datetime.now().strftime('%m-%d-%y') + '.log'
try:
if os.path.exists(self.logfile):
self.fd = BufferedWriter(FileIO(self.logfile, 'ab'))
else:
self.fd = BufferedWriter(FileIO(self.logfile, 'wb'))
except IOError:
self.log.error("Cannot create log file %s" % self.logfile)
try:
if os.path.exists(self.datafile):
self.fd = BufferedWriter(FileIO(self.logfile, 'ab'))
else:
self.fd = BufferedWriter(FileIO(self.logfile, 'wb'))
except IOError:
self.log.error("Cannot create data file %s" % self.datafile)
self.check_conditions(self.datafile)
time.sleep(timeInterval)
def __init__(self):
self.start_flying = True
self.stop_flying = False
self.return_to_home = False
self.is_takeoff = False
# self.PID = SimplePID()
self.drone_position_x = 0
self.drone_position_y = 0
self.drone_position_z = 0
self.drone_velocity_x = 0
self.drone_velocity_y = 0
self.drone_velocity_z = 0
self.drone_acceleration_x = 0
self.drone_acceleration_y = 0
self.drone_acceleration_z = 0
self.target_position_x = 0
self.target_position_y = 0
self.target_position_z = 0
self.target_velocity_x = 0
self.target_velocity_y = 0
self.target_velocity_z = 0
self.drone_yaw = 0
self.drone_yaw_radians = 0
self.vx = 0
self.vy = 0
self.vx1 = 0
self.vy1 = 0
self.ax = 0
self.ay = 0
self.controller = BasicDroneController()
self.subNavdata = rospy.Subscriber('/ardrone/navdata',Navdata,self.ReceiveNavdata)
self.logger = logging.getLogger('LQR_simulation')
self.fileHandler_message = logging.StreamHandler(BufferedWriter(FileIO("LQR_simulation_data" + time.strftime("%Y%m%d-%H%M%S") + ".log", "w")))
self.logger.addHandler(self.fileHandler_message)
self.formatter_message = logging.Formatter('%(message)s')
self.fileHandler_message.setFormatter(self.formatter_message)
self.logger.setLevel(LoggerWarningLevel)
self.logger.info('Time;target_position_x,target_position_y,target_position_z;target_velocity_x,target_velocity_y,target_velocity_z;drone_position_x,drone_position_y,drone_position_z;drone_velocity_x,drone_velocity_y,drone_velocity_z,vx1,vy1,ax,ay')
self.logger_land = logging.getLogger('LQR_simulation_land')
self.fileHandler_message = logging.StreamHandler(BufferedWriter(FileIO("LQR_simulation_PD_land_data" + time.strftime("%Y%m%d-%H%M%S") + ".log", "w")))
self.logger_land.addHandler(self.fileHandler_message)
self.formatter_message = logging.Formatter('%(message)s')
self.fileHandler_message.setFormatter(self.formatter_message)
self.logger_land.setLevel(LoggerWarningLevel)
self.logger_land.info('Time;target_position_x,target_position_y,target_position_z;target_velocity_x,target_velocity_y,target_velocity_z;drone_position_x,drone_position_y,drone_position_z;drone_velocity_x,drone_velocity_y,drone_velocity_z,vx1,vy1,ax,ay')
def file_rollover(self, t):
""" rollover the file if necessary
"""
if self.rollover_time == None:
# Not initialized yet... set the current day and next rollover
self.rollover_time = self.rollover_ref + int(
(t - self.rollover_ref) / 86400) * 86400
if self.rollover_time <= t:
# Close the currently open file
if (self.fd != None):
self.log.info("Closing %s" % self.fd.name)
self.fd.close()
self.fd = None
Day = strftime("%Y_%m_%d", gmtime(self.rollover_time))
Year = Day[0:4]
fileBase = "%s_%s" % (self.filePrefix, Day)
yearPath = os.path.join(self.root_archive_dir, Year)
if not os.path.exists(yearPath):
try:
os.makedirs(yearPath)
except os.error as e:
self.log.error("Cannot create %s : %s" %
(yearPath, str(e)))
self.log.error("Redirecting to /tmp")
yearPath = os.path.join('/tmp', Year)
if not os.path.exist(yearPath):
try:
os.makedirs(yearPath)
except os.error as e:
self.log.error("Cannot create %s either: %s" %
(yearPath, str(e)))
if os.path.exists(yearPath):
filePath = os.path.join(yearPath, fileBase + ".log")
try:
if os.path.exists(filePath):
self.fd = BufferedWriter(FileIO(filePath, 'ab'))
else:
self.fd = BufferedWriter(FileIO(filePath, 'wb'))
except IOError:
self.log.error("Cannot create log file %s" % filePath)
if os.path.exists(yearPath) and self.raw:
filePath = os.path.join(yearPath, fileBase + ".raw")
try:
if os.path.exists(filePath):
self.fdRaw = BufferedWriter(FileIO(filePath, 'ab'))
else:
self.fdRaw = BufferedWriter(FileIO(filePath, 'wb'))
except IOError:
self.log.error("Cannot create log file %s" % filePath)
self.rollover_time += 86400
def makenew(self):
import os
import io
from io import FileIO, BufferedWriter
logging.debug(str(self.ts()) + 'Trying makenew.')
try:
with BufferedWriter(FileIO(self.output, 'wb'),
buffer_size=4096000) as self.outf:
logging.debug(str(self.ts()) + 'Trying searchfiles.')
self.searchfiles(self.filelist, self.findlist)
logging.debug(str(self.ts()) + 'Finished searchfiles.')
self.outf.close
print("Total Lines: " + str(self.totline))
print("Total time: " + str(self.tottime))
logging.info(str(self.ts()) + "Total Lines: " + str(self.totline))
logging.info(str(self.ts()) + "Total time: " + str(self.tottime))
except:
logging.error(str(self.ts()) + 'Something went terribly wrong.',
exc_info=True)
def predict(request):
"""
Get a prediction on the image put image.
"""
if request.method == 'POST':
# reset keras session
keras.backend.clear_session()
# get the image file
img = request.FILES['predict_image']
with BufferedWriter(FileIO('tmp.jpg', 'w')) as tmp_img:
for chunk in img.chunks():
tmp_img.write(chunk)
# read file as black and white
img = imread('./tmp.jpg')[:, :, :1] / 255.0
img = resize(img, (28, 28), mode='constant', preserve_range=True)
img = np.expand_dims(img, axis=0)
# make sure the image shape is right
assert img.shape == (1, 28, 28, 1)
# invert image
img = 1 - img
# load model
_, model, _ = CapsNet(input_shape=(28, 28, 1), n_class=10, routings=3)
model.load_weights('./capsnet/trained_model.h5')
# predict
predictions, _ = model.predict(img, batch_size=100)
prediction = predictions[0]
prediction_number = np.argmax(prediction)
return JsonResponse({'prediction': np.asscalar(prediction_number)},
status=200)
def saveUpload(uploaded, filename, raw_data):
'''
raw_data: if True, uploaded is an HttpRequest object with the file being
the raw post data
if False, uploaded has been submitted via the basic form
submission and is a regular Django UploadedFile in request.FILES
'''
#filename = settings.MEDIA_ROOT + '/documents/GenBank/'+ filename
filename = settings.MEDIA_ROOT + '/' + filename
try:
from io import FileIO, BufferedWriter
with BufferedWriter(FileIO(filename, "wb")) as dest:
# if the "advanced" upload, read directly from the HTTP request
# with the Django 1.3 functionality
if raw_data:
foo = uploaded.read(1024)
while foo:
dest.write(foo)
foo = uploaded.read(1024)
# if not raw, it was a form upload so read in the normal Django chunks fashion
else:
for c in uploaded.chunks():
dest.write(c)
# got through saving the upload, report success
return True
except IOError:
# could not open the file most likely
return HttpResponseBadRequest("Could not open the file")
return False
def handle(self):
"""
Handle a single HTTP request. Shamelessly copied from Python
3.5 wsgiref simple_server. Adjusted the SimpleHandler to set
multithread=False.
"""
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ''
self.request_version = ''
self.command = ''
self.send_error(414)
return
if not self.parse_request(): # an error code has been sent, exit
return
# Avoid passing the raw file object wfile, which can do partial
# writes (Issue 24291)
stdout = BufferedWriter(self.wfile)
try:
handler = MySimpleHandler(
self.rfile, stdout, self.get_stderr(), self.get_environ(),
multithread=False, multiprocess=False)
handler.request_handler = self # backpointer for logging
handler.run(self.server.get_app())
finally:
stdout.detach()
def setup(self, filename, *args, **kwargs):
self._path = os.path.join('tracker/media', self.UPLOAD_DIR, filename)
try:
os.makedirs(os.path.realpath(os.path.dirname(self._path)))
except:
pass
self._dest = BufferedWriter(FileIO(self._path, "a"))
def post_send(request):
user = check_auth(request)
if not user:
return HttpResponseForbidden('error in user validation')
if request.method != 'POST':
return HttpResponseBadRequest('bad request post')
form = PinDirectForm(request.POST, request.FILES)
if form.is_valid():
upload = request.FILES.values()[0]
filename = create_filename(upload.name)
try:
with BufferedWriter(
FileIO("%s/pin/images/o/%s" % (MEDIA_ROOT, filename),
"wb")) as dest:
for c in upload.chunks():
dest.write(c)
model = Post()
model.image = "pin/images/o/%s" % (filename)
model.user = user
model.timestamp = time.time()
model.text = form.cleaned_data['description']
model.category_id = form.cleaned_data['category']
model.device = 2
model.save()
return HttpResponse('success')
except IOError:
return HttpResponseBadRequest('error')
return HttpResponseBadRequest('bad request in form')
else:
HttpResponseBadRequest('error in form validation')
return HttpResponseBadRequest('bad request')
def score(graphs, schema, url, port):
"""
graphs is expected to be a list of dictionaries, where each entry in the
list represents a graph with
* key idx -> index value
* key nodes -> list of ints representing vertices of the graph
* key edges -> list of list of ints representing edges of graph
"""
stream = BufferedWriter(BytesIO())
writer = DataFileWriter(stream, avro.io.DatumWriter(), schema)
# writer = DataFileWriter(open("imdb-graph.avro", "wb"), DatumWriter(), schema)
for graph in graphs:
writer.append({
"edges": graph["edges"],
"vertices": graph["vertices"],
"index": graph["idx"],
"label": graph.get("label")
})
writer.flush()
raw_bytes = stream.raw.getvalue()
writer.close()
url = "{}:{}/predictUnstructured/?ret_mode=binary".format(
url.strip("/"), port)
payload = raw_bytes
headers = {'Content-Type': 'application/octet-stream'}
response = requests.request("POST", url, headers=headers, data=payload)
return response
def save(root, writer, options=None):
options = options.copy() if options else Options()
report = options.report
with ProgressBar('save', len(root._context.mutable)) as progress:
with BufferedWriter(writer, 0x1000) as writer:
options.report = lambda x: (report(x), progress.report(x))
return write(root, writer, options)
def __init__(self, stream=None):
if not stream:
stream = BytesIO()
self.stream = stream
self.writer = BufferedWriter(self.stream)
self.written_count = 0
def save_uploaded_from_request(uploaded, extension, raw_data):
"""
raw_data: if True, upfile is a HttpRequest object with raw post data
as the file, rather than a Django UploadedFile from request.FILES
"""
filename = "/tmp/%s.%s" % (uuid4(), extension)
try:
with BufferedWriter(FileIO(filename, "wb")) as dest:
# if the "advanced" upload, read directly from the HTTP request
# with the Django 1.3 functionality
if raw_data:
foo = uploaded.read(1024)
while foo:
dest.write(foo)
foo = uploaded.read(1024)
# if not raw, it was a form upload so read in the normal Django chunks fashion
else:
for c in uploaded.chunks():
dest.write(c)
return filename
except IOError:
# could not open the file most likely
return False
except Exception, e:
return str(e)
def read(self, size):
"""
Reads (receives) a whole block of size bytes from the connected peer.
:param size: the size of the block to be read.
:return: the read data with len(data) == size.
"""
if self._socket is None:
self._raise_connection_reset()
# TODO Remove the timeout from this method, always use previous one
with BufferedWriter(BytesIO(), buffer_size=size) as buffer:
bytes_left = size
while bytes_left != 0:
try:
partial = self._socket.recv(bytes_left)
except socket.timeout as e:
raise TimeoutError() from e
except ConnectionError:
self._raise_connection_reset()
except OSError as e:
if e.errno in CONN_RESET_ERRNOS:
self._raise_connection_reset()
else:
raise
if len(partial) == 0:
self._raise_connection_reset()
buffer.write(partial)
bytes_left -= len(partial)
# If everything went fine, return the read bytes
buffer.flush()
return buffer.raw.getvalue()
def save_upload(uploaded, filename, raw_data):
"""
raw_data: if True, uploaded is an HttpRequest object with the file being
the raw post data
if False, uploaded has been submitted via the basic form
submission and is a regular Django UploadedFile in request.FILES
"""
try:
with BufferedWriter(FileIO(filename, "wb")) as dest:
# if the "advanced" upload, read directly from the HTTP request
# with the Django 1.3 functionality
if raw_data:
foo = uploaded.read(1024)
while foo:
dest.write(foo)
foo = uploaded.read(1024)
# if not raw, it was a form upload so read in the normal Django chunks fashion
else:
for c in uploaded.chunks():
dest.write(c)
# got through saving the upload, report success
return True
except IOError:
# could not open the file most likely
pass
return False
def write_orc_partition(df, path, fs, filename, compression=None):
full_path = fs.sep.join([path, filename])
with fs.open(full_path, mode="wb") as out_file:
if not isinstance(out_file, IOBase):
out_file = BufferedWriter(out_file)
cudf.io.to_orc(df, out_file, compression=compression)
return full_path
def save_upload(uploaded, filename, raw_data, user):
'''
raw_data: if True, uploaded is an HttpRequest object with the file being
the raw post data
if False, uploaded has been submitted via the basic form
submission and is a regular Django UploadedFile in request.FILES
'''
file_ext = os.path.splitext(filename)[1]
file_name = os.path.splitext(filename)[0]
filename = safe_ascii(file_name) + file_ext
path = make_temp_directory(filename, user)
public_path = path.replace(MEDIA_ROOT, '', 1)
# try:
with BufferedWriter(FileIO(path, "wb")) as dest:
# if the "advanced" upload, read directly from the HTTP request
# with the Django 1.3 functionality
if raw_data:
foo = uploaded.read(1024)
while foo:
dest.write(foo)
foo = uploaded.read(1024)
# if not raw, it was a form upload so read in the normal Django chunks fashion
else:
for c in uploaded.chunks():
dest.write(c)
# got through saving the upload, report success
return public_path
# except IOError:
# could not open the file most likely
# pass
return False
def handle(self):
"""Handle a single HTTP request"""
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ''
self.request_version = ''
self.command = ''
self.send_error(414)
return
if not self.parse_request(): # An error code has been sent, just exit
return
# Avoid passing the raw file object wfile, which can do partial
# writes (Issue 24291)
stdout = BufferedWriter(self.wfile)
try:
handler = ServerHandler(
self.rfile, stdout, self.get_stderr(), self.get_environ()
)
handler.request_handler = self # backpointer for logging
handler.run(self.server.get_app())
finally:
stdout.detach()
def read(self, size):
"""Reads (receives) a whole block of 'size bytes
from the connected peer.
A timeout can be specified, which will cancel the operation if
no data has been read in the specified time. If data was read
and it's waiting for more, the timeout will NOT cancel the
operation. Set to None for no timeout
"""
if self._socket is None:
raise ConnectionResetError()
# TODO Remove the timeout from this method, always use previous one
with BufferedWriter(BytesIO(), buffer_size=size) as buffer:
bytes_left = size
while bytes_left != 0:
try:
partial = self._socket.recv(bytes_left)
except socket.timeout as e:
raise TimeoutError() from e
except OSError as e:
if e.errno == errno.EBADF or e.errno == errno.ENOTSOCK:
self._raise_connection_reset()
else:
raise
if len(partial) == 0:
self._raise_connection_reset()
buffer.write(partial)
bytes_left -= len(partial)
# If everything went fine, return the read bytes
buffer.flush()
return buffer.raw.getvalue()
def save_bits(file_name: str, bool_array: nparray, packed=True) -> int:
'''
Save bits to a file from a bool array.
Parameters
----------
file_name: string
The name of the file to save.
bool_array: numpy.array
The bool array.
packed: bool
Whether to pack the bits into bytes.
Defaults to True.
Returns the number of bytes saved.
'''
with open(file_name, 'wb') as bit_file:
writer = BufferedWriter(bit_file)
count = 0
if packed:
for byte in pack_bools_to_bytes(bool_array):
writer.write(byte)
count += 1
else:
for byte in bools_to_bytes(bool_array):
writer.write(byte)
count += 1
writer.flush()
return count
def setup(self, filename, *args, **kwargs):
self._path = self.get_path(filename, *args, **kwargs)
try:
os.makedirs(os.path.realpath(os.path.dirname(self._path)))
except:
pass
self._dest = BufferedWriter(FileIO(self._path, "w"))
def setup(self, filename):
self._path = os.path.join(settings.MEDIA_ROOT, filename)
try:
os.makedirs(os.path.realpath(os.path.dirname(self._path)))
except:
pass
self._dest = BufferedWriter(FileIO(self._path, "w"))
def get_or_init_buffered_writer(self, file):
try:
if self.buffered_writer is None or self.buffered_writer.closed:
self.buffered_writer = BufferedWriter(FileIO(self.settings.get_data_folder_path() + file, 'a'))
return self.buffered_writer
except IOError as e:
log.error("Failed to initialize buffered writer! Error: %s", e)
raise RuntimeError("Failed to initialize buffered writer!", e)
def write_file(self, data):
log.info('Writing data to %s', self.file.filepath)
try:
with BufferedWriter(FileIO(self.file.filepath, "wb")) as output:
output.write(data)
output.close()
except Exception:
log.exception('Failed to write file')
def writeLexicon():
with BufferedWriter(FileIO("../data/Lexicon.txt", "a")) as outLexFile:
for item in lexiconBuffer:
outLexFile.write(item[0] + ':' + str(item[1]) + ',' + str(item[2]))
outLexFile.write('\n')
del lexiconBuffer[:]
outLexFile.close()
return
def setup(self, filename):
ext = os.path.splitext(filename)[1]
self._filename = md5(filename.encode('utf8')).hexdigest() + ext
self._path = os.path.join(self._upload_dir, self._filename)
try:
os.makedirs(os.path.realpath(os.path.dirname(self._path)))
except:
pass
self._destination = BufferedWriter(FileIO(self._path, "w"))
def __init__(self, fobj, mode='rb', bufsize=-1, close=True):
"""
:param fobj: Either an integer fileno, or an object supporting the
usual :meth:`socket.fileno` method. The file will be
put in non-blocking mode.
"""
if isinstance(fobj, int):
fileno = fobj
fobj = None
else:
fileno = fobj.fileno()
if not isinstance(fileno, int):
raise TypeError('fileno must be int: %r' % fileno)
orig_mode = mode
mode = (mode or 'rb').replace('b', '')
if 'U' in mode:
self._translate = True
mode = mode.replace('U', '')
else:
self._translate = False
if len(mode) != 1:
# Python 3 builtin `open` raises a ValueError for invalid modes;
# Python 2 ignores in. In the past, we raised an AssertionError, if __debug__ was
# enabled (which it usually was). Match Python 3 because it makes more sense
# and because __debug__ may not be enabled
raise ValueError('mode can only be [rb, rU, wb], not %r' %
(orig_mode, ))
self._fobj = fobj
self._closed = False
self._close = close
self.fileio = GreenFileDescriptorIO(fileno, mode, closefd=close)
if bufsize < 0:
bufsize = self.default_bufsize
if mode == 'r':
if bufsize == 0:
bufsize = 1
elif bufsize == 1:
bufsize = self.default_bufsize
self.io = BufferedReader(self.fileio, bufsize)
elif mode == 'w':
if bufsize == 0:
bufsize = 1
elif bufsize == 1:
bufsize = self.default_bufsize
self.io = BufferedWriter(self.fileio, bufsize)
else:
# QQQ: not used
self.io = BufferedRandom(self.fileio, bufsize)
if self._translate:
self.io = TextIOWrapper(self.io)
def __init__(self, stream=None, known_length=None):
if not stream:
stream = BytesIO()
if known_length is None:
# On some systems, DEFAULT_BUFFER_SIZE defaults to 8192
# That's over 16 times as big as necessary for most messages
known_length = max(DEFAULT_BUFFER_SIZE, 1024)
self.writer = BufferedWriter(stream, buffer_size=known_length)
self.written_count = 0
def read(self, size, timeout=timedelta(seconds=5)):
"""Reads (receives) a whole block of 'size bytes
from the connected peer.
A timeout can be specified, which will cancel the operation if
no data has been read in the specified time. If data was read
and it's waiting for more, the timeout will NOT cancel the
operation. Set to None for no timeout
"""
# Ensure that only one thread can receive data at once
with self._lock:
# Ensure it is not cancelled at first, so we can enter the loop
self.cancelled.clear()
# Set the starting time so we can
# calculate whether the timeout should fire
start_time = datetime.now() if timeout else None
with BufferedWriter(BytesIO(), buffer_size=size) as buffer:
bytes_left = size
while bytes_left != 0:
# Only do cancel if no data was read yet
# Otherwise, carry on reading and finish
if self.cancelled.is_set() and bytes_left == size:
raise ReadCancelledError()
try:
partial = self._socket.recv(bytes_left)
if len(partial) == 0:
raise ConnectionResetError(
'The server has closed the connection (recv() returned 0 bytes).'
)
buffer.write(partial)
bytes_left -= len(partial)
except BlockingIOError as error:
# No data available yet, sleep a bit
time.sleep(self.delay)
# Check if the timeout finished
if timeout:
time_passed = datetime.now() - start_time
if time_passed > timeout:
raise TimeoutError(
'The read operation exceeded the timeout.'
) from error
# If everything went fine, return the read bytes
buffer.flush()
return buffer.raw.getvalue()
def connect(self):
"""Rewrite of KQMLModule connect, only handles send_socket and output
connections"""
try:
self.send_socket = socket()
self.send_socket.connect((self.host, self.port))
socket_write = SocketIO(self.send_socket, 'w')
self.out = BufferedWriter(socket_write)
except OSError as error_msg:
LOGGER.error('Connection failed: %s', error_msg)
# Verify that you can send messages...
assert self.out is not None, \
'Connection formed but output (%s) not set.' % (self.out)
