def main(dataPath, datacollector, display):
"""Main function of the odb2collector.
:param String dataPath: Directory where the data will be stored.
:param DataCollector datacollector: DataCollector instance used.
:param Display display: Display instance used to show messages.
"""
## Numer of bytes stored
loggedBytes = 0
## Initiate the compressor
gzip = Compressor()
gzip.start()
logEnding = CONFIGURATION["logending"]
while True:
message = [
"Logged %skB" % (loggedBytes / 1024)
]
print "\n".join(message)
display.write_message(message)
datafile = "%s/%s.%s" % (dataPath, _getCurrentTime(), logEnding)
loggedBytes += datacollector.write_data_log(
datafile,
nbrOfOBDFrames=1000,
messagesPerTimestamp=20
)
print "Collected data log..."
gzip.add_file_for_compression(datafile)
def compress(self, data, file_name, k):
compressed_file = open(
f'./data/large_inputs/compression/{file_name}.bin', 'wb')
compressor = Compressor(data=data,
dictionary=self.init_code_dictionary(
k=k, file_type=file_name.split(".")[-1]),
k=k)
compressor_response = compressor.run()
elapsed_time = compressor_response["time"]
self.compressed_message = compressor_response["message"]
compressed_file.write(
struct.pack(f">{'H'*len(self.compressed_message)}",
*self.compressed_message))
compressed_file.close()
print(
f"\nSIZE AFTER COMPRESSION: {os.path.getsize(f'./data/large_inputs/compression/{file_name}.bin')}"
)
return {
"Compression file size":
os.path.getsize(
f'./data/large_inputs/compression/{file_name}.bin'),
"Elapsed Time":
elapsed_time,
"Indices":
len(self.compressed_message)
}
def rle_test(self): filename = "testdata.txt" comp = Compressor(self.datapath + filename) origdata = open(self.datapath + filename, 'rb').read() print "Begining RLE Test!", comp.rle_encode() assert origdata == comp.rle_decode() print "..Passed"
def decompress(self, compresses_index, compression_type='var_byte'):
self.index = dict()
for term in compresses_index.keys():
self.index[term] = dict()
for doc_id in compresses_index[term]:
if compression_type == 'var_byte':
self.index[term][doc_id] = Compressor.variable_byte_decode(
compresses_index[term][doc_id])
else:
self.index[term][doc_id] = Compressor.gamma_decode(
compresses_index[term][doc_id])
def compress(self, compression_type='var_byte'):
compressed_index = dict()
for term in self.index.keys():
compressed_index[term] = dict()
for doc_id in self.index[term]:
if compression_type == 'var_byte':
compressed_index[term][
doc_id] = Compressor.variable_byte_encode(
self.index[term][doc_id])
else:
compressed_index[term][doc_id] = Compressor.gamma_encode(
self.index[term][doc_id])
return compressed_index
def __init__(self):
self.compressor = Compressor()
self.turbine = Turbine()
self.initial_guess = self.__class__.DEFAULT_PARAMS.copy()
self.cpc = 1004 # J/(kg K)
self.cpt = 1225 # J/(kg K)
self.T01 = 273.15 # K
self.P01 = 101e3 # Pa
self.R_c = (1 - 1 / self.compressor.gam) * self.cpc # from gam and cp
self.R_t = (1 - 1 / self.turbine.gam) * self.cpt
self.A8 = pi / 4 * 35e-3**2 # Measured: D8 = 45mm
self.FHV = 43e6 #J/kg Fuel lower calorific value
self.eta_combustion = 0.5
def __init__(self,V=1000.,cmp_eff=80.,trb_eff=90.,p=200000):
# Air Side
self.amb = Ambient()
self.cmp = Compressor(eff=cmp_eff)
self.tank = Tank(V=V,p=p)
self.trb = Turbine(eff=trb_eff)
self.m_dot_i = 0.0
self.m_dot_o = 0.0
# Heat Side
# Create pandas DataFrame to store results
self.df = pd.DataFrame(columns=variables)
async def get_compiled_file(self, session: aiohttp.ClientSession,
target: BuildTarget, lock: asyncio.Lock,
compressor: Compressor):
print(f"Waiting for a compiled file from {self.address}")
async with lock:
data = aiohttp.FormData()
archive_file = tempfile.NamedTemporaryFile(suffix='tar.xz')
commands_file = tempfile.NamedTemporaryFile(suffix='.sh')
target.substitute_for_absolute_paths('/')
data.add_field('workdir', os.getcwd())
data.add_field('targets', ', '.join(target.target_files))
data.add_field('commands_file', commands_file.name)
data.add_field('password', self.password)
data.add_field('exact_lib_versions',
'true' if config.EXACT_LIB_VERSIONS else 'false')
if self.libraries is None:
self.libraries = []
response = await session.post(url=self.address +
self.ALL_LIBRARIES_ENDPOINT)
libraries = (await response.json())
for library in libraries:
try:
self.libraries.append(Library(library))
except:
continue
present_libraries = await self.get_present_libraries(target)
async with lock:
for library in present_libraries:
await target.replace_in_commands(
library, f'${{{library.split("/")[-1]}}}')
await target.create_commands_file(commands_file.name)
compressor.compress([
file for file in target.all_dependency_files
if file not in present_libraries
] + [commands_file.name], archive_file.name)
data.add_field('file',
open(archive_file.name, 'rb'),
filename=archive_file.name.split('/')[-1])
archive_file.close()
commands_file.close()
# Lock is released
result = await self.get_archive(data, session)
async with lock:
await Compressor.extract_files(result)
def main():
p = argparse.ArgumentParser()
p.add_argument('raw')
p.add_argument('compressed', nargs='*')
flags = p.parse_args()
if ';' in flags.raw:
flags.raw, flags.compressed = flags.raw.split(';')
else:
flags.compressed = flags.compressed[0]
raw = cached_listdir_imgs(flags.raw, discard_shitty=False)
compressed = cached_listdir_imgs(flags.compressed, discard_shitty=False)
print(raw, compressed)
print('Average bpp', np.mean([Compressor.bpp_from_compressed_file(p) for p in compressed.ps]))
r = ResidualDataset(compressed, raw, 'diff', 256, True, output_device=pe.DEVICE)
# for i in range(len(r)):
# print(r[i]['residual'].unique())
# exit(1)
d = DataLoader(r, batch_size=10, shuffle=False, num_workers=2)
mi, ma = None, None
for b in d:
res = b['raw']
# print(res.unique())
if mi is None:
mi = res.min()
ma = res.max()
else:
mi = min(res.min(), mi)
ma = max(res.max(), ma)
print(mi, ma)
def get(self, fp, off=0, size=-1):
"""
Get object by `fingerprint`
- Throws `ObjectNotFound` RADOS exception, if no object with that fingerprint exists
- Return decompressed object
- Return the whole object by default
- If you specify `off` and `size` it is applied only after fetching the whole object
"""
self.log.debug("GET [%r]: %s:%s", fp, off, size)
obj_size, _ = self.ioctx.stat(fp)
compression_id = CAS._convert_meta(
self.ioctx.get_xattr(fp, "cas.meta.compression"))
decompressor = Compressor.select(compression_id)
self.log.debug("GET [%r]: size %d compressed with %r", fp, obj_size,
compression_id)
compressed_data = self.ioctx.read(fp, obj_size, 0)
data = decompressor.decompress(compressed_data)
if size < 0:
size = len(data)
return data[off:off + size]
def _get_pils(self, idx):
if self._cache and idx in self._cache:
yield self._cache[idx]
return
compressed_p = self.compressed_images.ps[idx]
bpp = Compressor.bpp_from_compressed_file(compressed_p)
raw_p = self.compressed_to_raw[os.path.basename(compressed_p)]
compressed, f1 = self._read_img(compressed_p)
raw, f2 = self._read_img(raw_p)
if not NO_ERRORS:
assert compressed.size == raw.size, f'Error for {compressed_p}, {raw_p}; {compressed.size, raw.size}'
if self._cache is not None and idx not in self._cache:
with self._cache_lock:
if idx not in self._cache:
print('Caching', idx)
compressed, raw = map(ResidualDataset._convert, (compressed, raw))
self._cache[idx] = compressed, raw, bpp
yield compressed, raw, bpp
f1.close()
f2.close()
def get(self, fp, off=0, size=-1):
"""
Get object by `fingerprint`
- Throws `ObjectNotFound` RADOS exception, if no object with that fingerprint exists
- Return decompressed object
- Return the whole object by default
- If you specify `off` and `size` it is applied only after fetching the whole object
"""
self.log.debug("GET [%r]: %s:%s", fp, off, size)
obj_size, _ = self.ioctx.stat(fp)
compression_id = CAS._convert_meta(self.ioctx.get_xattr(fp, "cas.meta.compression"))
decompressor = Compressor.select(compression_id)
self.log.debug("GET [%r]: size %d compressed with %r", fp, obj_size, compression_id)
compressed_data = self.ioctx.read(fp, obj_size, 0)
data = decompressor.decompress(compressed_data)
if size < 0:
size = len(data)
return data[off:off+size]
def make_bin_pkl(img_folder, num_bins, overwrite=False):
if isinstance(img_folder, list):
img_folder = img_folder[0]
pkl_out = get_default_pkl_p(img_folder, num_bins)
if os.path.isfile(pkl_out):
return pkl_out
assert 'train_oi' in img_folder, img_folder # currently not supported, bc min_size and discard_shitty flags
ps = cached_listdir_imgs.cached_listdir_imgs(img_folder,
min_size=None,
discard_shitty=True).ps
ps_bpsps = sorted(
((p, Compressor.bpp_from_compressed_file(p) / 3) for p in ps),
key=lambda xy: xy[1]) # sort by bpsp
bpsps = [bpsp for _, bpsp in ps_bpsps]
# border b0 b1 ... bk k+1 borders
# bin_idx 0 1 2 ... k k+1 => k+2 bins
#
# for N bins, we need N-1 borders
# first border is after 1/NB-th of the data
# NB + 1 so that we get NB-1 evenly spaced bins *within* the data
bin_borders_x = np.linspace(0, len(bpsps) - 1, num_bins + 1, dtype=np.int)
# throw away the boundaries
bin_borders_x = bin_borders_x[1:-1]
bin_borders_b = [bpsps[x] for x in bin_borders_x]
with open(pkl_out, 'wb') as f:
print('Saving', bin_borders_b, '\n->', pkl_out)
pickle.dump(bin_borders_b, f)
return pkl_out
def dump_tables(fontfile, output):
font = TTFont(fontfile, lazy=True)
dump_folder = output + '_tables'
print('dump results in {0}'.format(dump_folder))
try:
os.makedirs(dump_folder)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
header_dict = FontInfo.getInformation(fontfile, FontInfo.TAGS.keys())
bin_header = BaseHeaderPrepare.prepare(BaseFonter.BASE_VERSION,
header_dict)
print('Base header total size=', len(bin_header))
base_fonter = BaseFonter(fontfile)
base_dump = dump_folder + '/base_dump'
base_fonter.dump_base(base_dump)
# OpenType tables.
dump_file = open(base_dump, 'r+b')
tables = font.reader.tables
for name in font.reader.tables:
table = tables[name]
offset = table.offset
length = table.length
#print('{0}: offset={1}, length={2}'.format(name, offset, length))
table_file_name = dump_folder + '/' + name.replace('/', '_')
table_file = open(table_file_name, 'w+b')
dump_file.seek(offset)
table_file.write(dump_file.read(length))
table_file.close()
rle_table = RleFont(table_file_name)
rle_table.encode()
rle_table.write(table_file_name)
compressor = Compressor(Compressor.GZIP_INPLACE_CMD)
compressor.compress(table_file_name)
print('{0}: offset={1:9d}\tlen={2:9d}\tcmp_len={3:9d}'.format(
name, offset, length, os.path.getsize(table_file_name + '.gz')))
print('TODO(bstell) save and compress the CFF parts.')
if 'CFF ' in font:
dumpCFFTable(font)
font.close()
def dump_tables(fontfile, output):
font = TTFont(fontfile,lazy=True)
dump_folder = output + '_tables'
print('dump results in {0}'.format(dump_folder))
try:
os.makedirs(dump_folder)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
header_dict = FontInfo.getInformation(fontfile, FontInfo.TAGS.keys())
bin_header = BaseHeaderPrepare.prepare(BaseFonter.BASE_VERSION, header_dict)
print('Base header total size=',len(bin_header))
base_fonter = BaseFonter(fontfile)
base_dump = dump_folder + '/base_dump'
base_fonter.dump_base(base_dump)
# OpenType tables.
dump_file = open(base_dump,'r+b')
tables = font.reader.tables
for name in font.reader.tables:
table = tables[name]
offset = table.offset
length = table.length
#print('{0}: offset={1}, length={2}'.format(name, offset, length))
table_file_name = dump_folder + '/' + name.replace('/', '_')
table_file = open(table_file_name,'w+b')
dump_file.seek(offset);
table_file.write(dump_file.read(length))
table_file.close()
rle_table = RleFont(table_file_name)
rle_table.encode()
rle_table.write(table_file_name)
compressor = Compressor(Compressor.GZIP_INPLACE_CMD)
compressor.compress(table_file_name)
print('{0}: offset={1:9d}\tlen={2:9d}\tcmp_len={3:9d}'.format(name, offset, length,os.path.getsize(table_file_name+'.gz')))
print('TODO(bstell) save and compress the CFF parts.')
if 'CFF ' in font:
dumpCFFTable(font)
font.close()
def _filter_compressed_images(self, raw_images):
"""
Find all compressed images that do not have a raw image, and remove them!
"""
raw_images_fn_to_p = {os.path.splitext(os.path.basename(p))[0] for p in raw_images.ps}
missing = set()
for p in self.compressed_images.ps:
if Compressor.filename_without_bpp(p) not in raw_images_fn_to_p:
missing.add(p)
if missing:
print(f'*** Missing {len(missing)} files that are in compressed but not in raw.')
self.compressed_images.ps = [p for p in self.compressed_images.ps if p not in missing]
def main(dataPath, datacollector, display):
"""Main function of the odb2collector.
:param String dataPath: Directory where the data will be stored.
:param DataCollector datacollector: DataCollector instance used.
:param Display display: Display instance used to show messages.
"""
## Numer of bytes stored
loggedBytes = 0
## Initiate the compressor
gzip = Compressor()
gzip.start()
logEnding = CONFIGURATION["logending"]
while True:
message = ["Logged %skB" % (loggedBytes / 1024)]
print "\n".join(message)
display.write_message(message)
datafile = "%s/%s.%s" % (dataPath, _getCurrentTime(), logEnding)
loggedBytes += datacollector.write_data_log(datafile,
nbrOfOBDFrames=1000,
messagesPerTimestamp=20)
print "Collected data log..."
gzip.add_file_for_compression(datafile)
def test_generateHeader(self):
myMap = {
ord('r'): '01',
ord('e'): '00',
ord('j'): '11',
ord('l'): '10',
}
chars = ['e', 'r', 'l', 'j']
payloadFormat = '001{0[0]}1{0[1]}01{0[2]}1{0[3]}'
expected = self.getHeader(chars, payloadFormat)
actual = Compressor(None, None).generateHeader(myMap)
self.assertEqual(expected, actual)
def base_font(self): output = self.folder + '/base' baseFonter = BaseFonter(self.fontfile) baseFonter.base(output) compressor = Compressor(Compressor.LZMA_CMD) compressor.compress(output, output + '.xz') compressor = Compressor(Compressor.GZIP_CMD) compressor.compress(output, output + '.gz')
def __init__(self, ioctx, compression="no"):
"""
Initialize CAS object. Caller needs to provide a connected and initialized
RADOS I/O context.
CAS objects need their own, exclusive RADOS I/O Context, since they operate on
objects in an extra namespace.
On initialization you can also specify the compression algorithm for
**new** objects. CAS never overwrites objects that already exists, but rather
increments their reference count. Existing objects have metadata to
select the right decompressor on `get`.
"""
self.ioctx = ioctx
self.ioctx.set_namespace("CAS")
self.compressor = Compressor.select(compression)
def __init__(self, ioctx, compression="no"):
"""
Initialize CAS object. Caller needs to provide a connected and initialized
RADOS I/O context.
CAS objects need their own, exclusive RADOS I/O Context, since they operate on
objects in an extra namespace.
On initialization you can also specify the compression algorithm for
**new** objects. CAS never overwrites objects that already exists, but rather
increments their reference count. Existing objects have metadata to
select the right decompressor on `get`.
"""
self.ioctx = ioctx
self.ioctx.set_namespace("CAS")
self.compressor = Compressor.select(compression)
def test_generateHeader3(self):
myMap = {
ord('b'): '100',
ord('p'): '01',
ord('e'): '00',
ord('o'): '101',
ord('r'): '1110',
ord('x'): '1111',
ord('!'): '1101',
ord(' '): '1100'
}
chars = ['e', 'p', 'b', 'o', ' ', '!', 'r', 'x']
payloadFormat = '001{0[0]}1{0[1]}001{0[2]}1{0[3]}001{0[4]}1{0[5]}01{0[6]}1{0[7]}'
expected = self.getHeader(chars, payloadFormat)
actual = Compressor(None, None).generateHeader(myMap)
self.assertEqual(expected, actual)
def build():
logger = logging.create_logger(app)
print(request.form)
password = request.form['password']
if not is_password_acceptable(password):
return make_response('', 400)
# Extracting files
tempdir = tempfile.TemporaryDirectory()
print(request.files)
f = request.files['file']
commands_file = request.form['commands_file'].strip('/')
archive_file = tempfile.NamedTemporaryFile(suffix='.tar.xz', dir=tempdir.name)
archive_filename = archive_file.name.split('/')[-1]
print(archive_filename)
f.save(archive_file.name)
compressor = Compressor(tempdir.name)
compressor.extract_files(archive_filename)
new_root = f'{tempdir.name}/{archive_filename.split(".")[0]}/'
archive_file.close()
# Run commands
print(new_root)
command_runner = CommandRunner(new_root + request.form['workdir'].strip('/'), app.config['LIBRARIES'])
output, code = command_runner.run_commands(new_root + commands_file, new_root, logger, request.form['exact_lib_versions'] == 'true')
if code != 0:
logger.debug(code)
response = output, 400
else:
# Sending files back
target_files = [target.strip('/') for target in request.form['targets'].split(', ')]
compressor.root_path = new_root
output_file = tempfile.NamedTemporaryFile(suffix='.tar.xz')
compressor.compress(target_files, output_file.name)
response = send_file(output_file, mimetype='application/x-object')
tempdir.cleanup()
return response
def _compressed_to_raw(compressed_images: Images, raw_images: Images):
"""
Create mapping compressed_image -> raw_image. Can be many to one!
"""
raw_images_fn_to_p = {os.path.splitext(os.path.basename(p))[0]: p for p in raw_images.ps}
compressed_to_raw_map = {}
errors = []
for p in compressed_images.ps:
try:
fn = Compressor.filename_without_bpp(p)
compressed_to_raw_map[os.path.basename(p)] = raw_images_fn_to_p[fn]
except KeyError as e:
errors.append(e)
if len(errors) > 100:
break
continue
if errors:
raise ValueError(f'Missing >={len(errors)} keys:', errors[:10],
f'\n{len(compressed_images.ps)} vs {len(raw_images.ps)}')
return compressed_to_raw_map
async def main():
# Check the file with GNU make
check_result = sp.Popen(f'make -n -f {config.MAKEFILE}',
stderr=sp.PIPE,
shell=True)
output = check_result.communicate()[1]
if check_result.returncode != 0:
print("File seems to be of bad format")
print(str(output))
exit(1)
lines = open(config.MAKEFILE, 'r').readlines()
p = Parser(lines)
p.replace_all_variables()
p.get_build_targets()
rm = await RequestsManager.create(p.default_target, hosts_file,
Compressor(), p)
try:
await rm.build_targets()
except ExecutionError as e:
print(e.commands_output, e.message)
from compressor import Compressor
compressor = Compressor("static/test.png")
print('')
print("====================")
print("= MATRICE =")
print("====================")
print('')
matrice = compressor.get_matrice()
for line in matrice:
print(line)
print('')
print("====================")
print("= ARRAY =")
print("====================")
print('')
print(compressor.get_array())
class Engine(Turbomachine):
def __init__(self):
self.compressor = Compressor()
self.turbine = Turbine()
self.initial_guess = self.__class__.DEFAULT_PARAMS.copy()
self.cpc = 1004 # J/(kg K)
self.cpt = 1225 # J/(kg K)
self.T01 = 273.15 # K
self.P01 = 101e3 # Pa
self.R_c = (1 - 1 / self.compressor.gam) * self.cpc # from gam and cp
self.R_t = (1 - 1 / self.turbine.gam) * self.cpt
self.A8 = pi / 4 * 35e-3**2 # Measured: D8 = 45mm
self.FHV = 43e6 #J/kg Fuel lower calorific value
self.eta_combustion = 0.5
DEFAULT_PARAMS = {
'P0_ratio_c': 1.2347241010782356,
'Mb_t': 0.27072466251896482,
'MFP4': 0.15310678698124691,
'MFP3': 0.14924987675288623,
'M_flight': 0,
'mdotf': 0.0019637316313999187,
'P0_ratio_t': 0.92680225375718472,
'MFP5': 0.15451547834023707,
'Mb_c': 0.44527731422329964,
'MFP': 0.14801196525452109,
'T0_ratio_t': 0.98261102832775471,
'T0_ratio_c': 1.0669702796449636,
'T04': 840.63888888888891
}
N_FREE_PARAMS = 2
def implicit_map(
self,
M_flight, # flight mach number
MFP,
MFP3,
Mb_c,
T0_ratio_c,
P0_ratio_c, # compressor
mdotf,
T04, # burner
MFP4,
MFP5,
Mb_t,
T0_ratio_t,
P0_ratio_t # turbine
):
"""
This function implements burner, nozzle and spool dynamics and integrates it with
turbine and compressor maps
Variable balance:
12 variables
- 11 equations
--------
2 free choices (e.g. M_flight and mdotf)
"""
# Expose constants
A8 = self.A8
FHV = self.FHV
eta_combustion = self.eta_combustion
A5 = self.turbine.geom['A2']
gam_c = self.compressor.gam
gam_t = self.turbine.gam
cpc = self.cpc
cpt = self.cpt
T01 = self.T01
P01 = self.P01
R_c = self.R_c
R_t = self.R_t
### Dimensionalize everything ###
dim_params = self.dimensionalize(
M_flight, # flight mach number
MFP,
MFP3,
Mb_c,
T0_ratio_c,
P0_ratio_c, # compressor
mdotf,
T04, # burner
MFP4,
MFP5,
Mb_t,
T0_ratio_t,
P0_ratio_t # turbine
)
T02 = dim_params.T02
T03 = dim_params.T03
T05 = dim_params.T05
P02 = dim_params.P02
P03 = dim_params.P03
P04 = dim_params.P04
P05 = dim_params.P05
omega_c = dim_params.omega_c
omega_t = dim_params.omega_t
mdot2 = dim_params.mdot2
mdot3 = dim_params.mdot3
mdot4 = dim_params.mdot4
mdot5 = dim_params.mdot5
### CONSTRAINTS ###
# * Energy addition in ther burner
res_T04 = T03 + mdotf * FHV * eta_combustion / (mdot3 * cpc) - T04
# * Spool has constant speed
res_omega = omega_c - omega_t
# * Conservation of energy in the spool
res_energy = mdot2 * cpc * (T03 - T02) - mdot4 * cpt * (T04 - T05)
# * Consevation of mass in the combustor
res_mdot = mdot3 + mdotf - mdot4
# * Nozzle exit is either choked or at ambient pressure
Pa = P01 / (1 + (gam_c - 1) / 2 * M_flight**2)**(gam_c / (gam_c - 1))
MFP8 = MFP5 * A5 / A8
res_MFP_nozzle = (P05 / Pa - 1) * MFP8 - (P05 / Pa - 1) * mach2mfp(
min(1, (2 / (gam_t - 1) * ((P05 / Pa)**(
(gam_t - 1) / gam_t) - 1))**0.5) if P05 / Pa > 1 else 0, gam_t)
### remove dimensions of residuals ###
# References for removing dimensions of residuals
mdotref = (T01 * R_c)**0.5 / (P01 * gam_c**0.5 *
self.compressor.geom['A1'])
h_ref = cpc * T01
omega_ref = (gam_c * R_c * T01)**0.5 / self.compressor.geom['D2']
#remove dimensions
res_omega /= omega_ref
res_energy /= h_ref * mdotref
res_mdot /= mdotref
res_T04 /= T04
### COMPRESSOR MAP ###
res_MFP_c, res_T0_ratio_c, res_P0_ratio_c = self.compressor.implicit_map(
MFP, MFP3, Mb_c, T0_ratio_c, P0_ratio_c, tol=1e-13)
### TURBINE MAP ###
res_MFP_t, res_T0_ratio_t, res_P0_ratio_t = self.turbine.implicit_map(
MFP4, MFP5, Mb_t, T0_ratio_t, P0_ratio_t, tol=1e-13)
return (res_omega, res_energy, res_mdot, res_MFP_nozzle, res_T04,
res_MFP_c, res_T0_ratio_c, res_P0_ratio_c, res_MFP_t,
res_T0_ratio_t, res_P0_ratio_t)
def dimensionalize(
self,
M_flight, # flight mach number
MFP,
MFP3,
Mb_c,
T0_ratio_c,
P0_ratio_c, # compressor
mdotf,
T04, # burner
MFP4,
MFP5,
Mb_t,
T0_ratio_t,
P0_ratio_t # turbine
):
gam_c = self.compressor.gam
gam_t = self.turbine.gam
T01 = self.T01
P01 = self.P01
R_c = self.R_c
R_t = self.R_t
### Dimensionalize everything ###
a01 = (gam_c * R_c * T01)**0.5
a04 = (gam_t * R_t * T04)**0.5
T02 = T01
T03 = T02 * T0_ratio_c
T04 = T04
T05 = T04 * T0_ratio_t
P02 = P01
P03 = P02 * P0_ratio_c
P04 = P03
P05 = P03 * P0_ratio_t
omega_c = Mb_c * a01 / (self.compressor.geom['D2'] / 2)
omega_t = Mb_t * a04 / (self.turbine.geom['D1t'] / 2)
mdot2 = MFP * self.compressor.geom['A1'] * P01 * gam_c**0.5 / (
T01 * R_c)**0.5
mdot3 = MFP3 * self.compressor.geom['A2'] * P03 * gam_c**0.5 / (
T03 * R_c)**0.5
mdot4 = MFP4 * self.turbine.geom['A1'] * P04 * gam_t**0.5 / (T04 *
R_t)**0.5
mdot5 = MFP5 * self.turbine.geom['A2'] * P05 * gam_t**0.5 / (T05 *
R_t)**0.5
#Nozzle
MFP8 = min(MFP5 * self.turbine.geom['A2'] / self.A8,
mach2mfp(1, gam_t))
M8 = mfp2mach(MFP8, gam_t)
P8 = P05 * (1 + (gam_t - 1) / 2 * M8**2)**(-gam_t / (gam_t - 1))
T8 = T05 * (1 + (gam_t - 1) / 2 * M8**2)**(-1)
dim_params = DimensionalParameters(T02=T02,
T03=T03,
T04=T04,
T05=T05,
T8=T8,
P02=P02,
P03=P03,
P04=P04,
P05=P05,
P8=P8,
omega_c=omega_c,
omega_t=omega_t,
mdot2=mdot2,
mdot3=mdot3,
mdot4=mdot4,
mdot5=mdot5)
return dim_params
def working_line(self, T04_grid):
wline = []
params = self.initial_guess
for T04 in T04_grid:
sol = self.general_explicit_map({
'M_flight': 0,
'T04': T04
}, params)
params = sol.params
wline.append(params)
return wline
def thrust(self, **params):
gam_t = self.turbine.gam
R_t = self.R_t
MFP8 = min(params['MFP5'] * self.turbine.geom['A2'] / self.A8,
mach2mfp(1, gam_t))
mdot = params['MFP5'] * self.turbine.geom['A2'] * params[
'P05'] * gam_t**0.5 / (params['T05'] * R_t)**0.5
M8 = mfp2mach(MFP8, gam_t)
P8 = params['P05'] * (1 + (gam_t - 1) / 2 * M8**2)**(-gam_t /
(gam_t - 1))
mdot5 = params['MFP5'] * self.turbine.geom['A2'] * params[
'P05'] * gam_t**0.5 / (params['T05'] * R_t)**0.5
T8 = params['T05'] * (1 + (gam_t - 1) / 2 * M8**2)**(-1)
return mdot * M8 * (gam_t * R_t * T8)**0.5 + (
P8 - self.P01 /
(1 + (gam_t - 1) * params['M_flight']**2)**0.5) * self.A8
def ode_fun(self, t, y):
P4, mdot, omega = y
self.compressor.explicit_map(mdot, omega)
T04 = self.throtle(t, mdot, T03)
self.turbine.explicit_map(P4, omega)
P4_dot = a01**2 * (mdot - mdott)
mdotdot = A1 / Lc * (P3 - P4)
omega_dot = 1 / J * torque_t - torque_c
return P4_dot, mdotdot, omega_dot
return total_work / self.eta
else:
return self.compressor_in.work_done / self.eta
except AttributeError as e:
raise e
if __name__ == '__main__':
from inlet import Inlet
from fan import Fan
from bypass import Bypass
from compressor import Compressor
ambient_conditions = FlowCondition(corrected_mass_flow=1400.,
mach=0.8, t_static=216, p_static=22632, station_number='1', medium='air')
inlet = Inlet(ambient=ambient_conditions, eta=0.98)
fan = Fan(inflow=inlet.outflow, eta=0.92, pressure_ratio=1.6, station_number='21')
bypass = Bypass(inflow=fan.outflow, bypass_ratio=8.)
lpc = Compressor(inflow=bypass.outflow_core, eta=0.9, pressure_ratio=1.4, station_number='25')
hpc = Compressor(inflow=lpc.outflow, eta=0.9, pressure_ratio=19, station_number='3')
lp_spool = Spool(compressor_in=(fan, lpc), eta=0.99)
print(lp_spool.work_required)
print(lpc.work_done)
print(hpc.inflow.mass_flow)
print(hpc.inflow.t_total)
print(hpc.p_total)
print(hpc.t_total)
print(hpc.work_done)
# print(obj.p_total)
# print(obj.t_total)
def main():
try:
init_options()
except OptionsError as e:
sys.stderr.write("Error: {0}\n\n".format(e))
sys.stderr.write(usage())
sys.stderr.write("\n");
sys.stderr.flush()
sys.exit(os.EX_CONFIG)
if options.main.generate_config:
print(generate_sample_config()) # XXX: this doesn't yet work properly because of groper
sys.exit()
conf_root = os.path.dirname(os.path.abspath(options.main.config))
facility_db = FacilityDB()
try:
facility_db.load_config(normalize_path(options.main.facilities_config, conf_root))
except (IOError) as e:
sys.stderr.write("Error reading {0}: {1}\n".format(options.main.facilities_config, e))
sys.stderr.flush()
sys.exit(os.EX_CONFIG)
except (FacilityError, configparser.Error) as e:
sys.stderr.write("{0} contains errors:\n\n".format(options.main.facilities_config))
if hasattr(e, 'lineno'):
e = 'Error on line {0}: {1}'.format(e.lineno, e.message.split("\n")[0].strip())
sys.stderr.write("{0}\n\n".format(e))
sys.stderr.write("Exiting now.\n")
sys.stderr.flush()
sys.exit(os.EX_CONFIG)
if options.main.check_config:
sys.exit() # We are just checking the config file, so exit here.
cache_config_checksum()
create_dirs()
if options.main.daemon:
daemonize()
if options.main.user:
drop_privileges(options.main.user)
if options.main.pidfile:
write_pid(options.main.pidfile)
atexit.register(exit_handler)
setup_logging()
try:
logging.getLogger().info("Starting loghogd.")
compressor = Compressor()
compressor.find_uncompressed(options.main.logdir, r'.+\.log\..+')
writer = Writer(facility_db, compressor, options.main.logdir)
processor = Processor(facility_db, writer)
server = Server(processor.on_message, conf_root)
signal_handler = make_shutdown_handler(server, writer, compressor)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGHUP, make_reload_handler(facility_db, writer))
except Exception as e:
logging.getLogger().error(e)
logging.getLogger().error('Exiting abnormally due to an error at startup.')
sys.exit(os.EX_CONFIG)
try:
compressor.start()
server.run()
except Exception as e:
logging.getLogger().exception(e)
logging.getLogger().error('Exiting abnormally due to an error at runtime.')
shutdown(None, server, writer, compressor)
sys.exit(os.EX_SOFTWARE)
logging.getLogger().info('Shutdown complete. Exiting.')
from compressor import Compressor from decompressor import Decompressor inputFileName = 'debug.txt' compressedFileName = 'compress.zip' decompressedFileName = 'uncompress.txt' Compressor(inputFileName, compressedFileName).run() Decompressor(compressedFileName, decompressedFileName).run()
from server import Server
from parse import Parser
from proxy_server import Proxy
print("SSSSSSSSSSSSSSSSSSSSSSSSS")
# initiate server object.
server = Server()
print("PPPPPPPPPPPPPPPPPPPPPPPPPPPP")
# initiate parser object.
parsers = Parser()
print("SSSSSSSSCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC")
# initiate scraper object.
scrape = Scraper()
print("CoCoCoCoCoCoCoCoCoCoCo")
# initiate compressor object.
compressor = Compressor()
print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")
# creates a connection pool to Server database.
server_mongodb = MongoDB(db_name='Server')
print("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
# creates a connection pool to Compressor database.
compressor_mongodb = MongoDB(db_name='Compressor')
print("YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY")
# counter and document id variables initial values.
crawler_doc_id = 0
count_crawler = 0
compressor_doc_id = 0
mongodb_doc_id = 0
#doc_id = 0
def main():
try:
init_options()
except OptionsError as e:
sys.stderr.write("Error: {0}\n\n".format(e))
sys.stderr.write(usage())
sys.stderr.write("\n")
sys.stderr.flush()
sys.exit(os.EX_CONFIG)
if options.main.generate_config:
print(generate_sample_config()
) # XXX: this doesn't yet work properly because of groper
sys.exit()
conf_root = os.path.dirname(os.path.abspath(options.main.config))
facility_db = FacilityDB()
try:
facility_db.load_config(
normalize_path(options.main.facilities_config, conf_root))
except (IOError) as e:
sys.stderr.write("Error reading {0}: {1}\n".format(
options.main.facilities_config, e))
sys.stderr.flush()
sys.exit(os.EX_CONFIG)
except (FacilityError, configparser.Error) as e:
sys.stderr.write("{0} contains errors:\n\n".format(
options.main.facilities_config))
if hasattr(e, 'lineno'):
e = 'Error on line {0}: {1}'.format(
e.lineno,
e.message.split("\n")[0].strip())
sys.stderr.write("{0}\n\n".format(e))
sys.stderr.write("Exiting now.\n")
sys.stderr.flush()
sys.exit(os.EX_CONFIG)
if options.main.check_config:
sys.exit() # We are just checking the config file, so exit here.
cache_config_checksum()
create_dirs()
if options.main.daemon:
daemonize()
if options.main.user:
drop_privileges(options.main.user)
if options.main.pidfile:
write_pid(options.main.pidfile)
atexit.register(exit_handler)
setup_logging()
try:
logging.getLogger().info("Starting loghogd.")
compressor = Compressor()
compressor.find_uncompressed(options.main.logdir, r'.+\.log\..+')
writer = Writer(facility_db, compressor, options.main.logdir)
processor = Processor(facility_db, writer)
server = Server(processor.on_message, conf_root)
signal_handler = make_shutdown_handler(server, writer, compressor)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGHUP, make_reload_handler(facility_db, writer))
except Exception as e:
logging.getLogger().error(e)
logging.getLogger().error(
'Exiting abnormally due to an error at startup.')
sys.exit(os.EX_CONFIG)
try:
compressor.start()
server.run()
except Exception as e:
logging.getLogger().exception(e)
logging.getLogger().error(
'Exiting abnormally due to an error at runtime.')
shutdown(None, server, writer, compressor)
sys.exit(os.EX_SOFTWARE)
logging.getLogger().info('Shutdown complete. Exiting.')
def learn_sparse_embedding(session,
model,
verbose=False,
output_frequency=500):
embedding_param = session.run(model.embedding)
embedding_basis = embedding_param[:model.
basis_size, :] # Choose first model.basis_size rows as basis
embedding_sp_ids_val = np.zeros(model.vocab_size_in * model.sparsity,
dtype=np_index_data_type())
embedding_sp_weights_val = np.zeros(model.vocab_size_in * model.sparsity)
# For basis vectors, they have one-hot expressions
for i in range(model.basis_size):
random_basis = {i: True}
j = 1 # j: number of elements in random_basis
while j < model.sparsity:
r = random.randint(0, model.basis_size - 1)
if (r not in random_basis):
random_basis[r] = True
j += 1
del random_basis[i]
# 1 true basis, model.sparsity - 1 paddings
indices = np.r_[
np.array([i], dtype=np_index_data_type()),
np.array(random_basis.keys(), dtype=np_index_data_type())]
embedding_sp_ids_val[i * model.sparsity:(i + 1) *
model.sparsity] = indices
# other weights are zero by the definition of `softmax_sp_weights_val`
embedding_sp_weights_val[i * model.sparsity] = 1.0
t1 = time.time()
# print("max non zero entry: ", model.sparsity)
# columns are bases. Need to transpose.
compressor = Compressor(bases=embedding_basis.T,
max_non_zero_entry=model.sparsity)
pool = multiprocessing.Pool(processes=FLAGS.cpu_count)
parallel_params = [(compressor, embedding_param[i, :])
for i in range(model.basis_size, model.vocab_size_in)]
results = pool.map(fit_wrapper, parallel_params)
for i_, result in enumerate(results):
indices, values = result
i = i_ + model.basis_size
embedding_sp_ids_val[i * model.sparsity:(i + 1) *
model.sparsity] = indices
embedding_sp_weights_val[i * model.sparsity:(i + 1) *
model.sparsity] = values
t2 = time.time()
print("Parallel time: ", t2 - t1)
finetune_save_path = os.path.join(FLAGS.save_path,
"finetune-" + FLAGS.model_config)
if not os.path.isdir(finetune_save_path):
os.mkdir(finetune_save_path)
sparse_parameters_path = finetune_save_path
np.save(os.path.join(sparse_parameters_path, "embedding_sp_ids_val.npy"),
embedding_sp_ids_val)
np.save(
os.path.join(sparse_parameters_path, "embedding_sp_weights_val.npy"),
embedding_sp_weights_val)
model.assign_sparse_embedding_params(session, embedding_basis)
return embedding_basis, embedding_sp_ids_val, embedding_sp_weights_val
def compress_files(self, files):
compressor = Compressor(files, self.runner.results_path,
debug=self.debug)
return compressor.zip()
def test_processFile4(self):
lastByte = '1000001'
expected = [130, 1]
actual = Compressor(None, None).processEOF(lastByte, [])
self.assertEqual(expected, actual)
import matplotlib.pyplot as plt
from compressor import Compressor
fig, ax = plt.subplots(1, 1, figsize=(4, 3))
c = Compressor()
c.plot_map(ax, samples=144)
plt.savefig('compressor_map.pdf')
import matplotlib.pyplot as plt
plt.rc('text', usetex=True)
plt.rc('font', family='serif')
from compressor import Compressor
plt.figure(figsize=(6,8),dpi=150)
Compressor().plot_map(plt.gca(), samples=100, plot=True)
plt.savefig('map.pdf', dpi=300)
plt.show()
