def main(dataPath, datacollector, display): """Main function of the odb2collector. :param String dataPath: Directory where the data will be stored. :param DataCollector datacollector: DataCollector instance used. :param Display display: Display instance used to show messages. """ ## Numer of bytes stored loggedBytes = 0 ## Initiate the compressor gzip = Compressor() gzip.start() logEnding = CONFIGURATION["logending"] while True: message = [ "Logged %skB" % (loggedBytes / 1024) ] print "\n".join(message) display.write_message(message) datafile = "%s/%s.%s" % (dataPath, _getCurrentTime(), logEnding) loggedBytes += datacollector.write_data_log( datafile, nbrOfOBDFrames=1000, messagesPerTimestamp=20 ) print "Collected data log..." gzip.add_file_for_compression(datafile)
def compress(self, data, file_name, k): compressed_file = open( f'./data/large_inputs/compression/{file_name}.bin', 'wb') compressor = Compressor(data=data, dictionary=self.init_code_dictionary( k=k, file_type=file_name.split(".")[-1]), k=k) compressor_response = compressor.run() elapsed_time = compressor_response["time"] self.compressed_message = compressor_response["message"] compressed_file.write( struct.pack(f">{'H'*len(self.compressed_message)}", *self.compressed_message)) compressed_file.close() print( f"\nSIZE AFTER COMPRESSION: {os.path.getsize(f'./data/large_inputs/compression/{file_name}.bin')}" ) return { "Compression file size": os.path.getsize( f'./data/large_inputs/compression/{file_name}.bin'), "Elapsed Time": elapsed_time, "Indices": len(self.compressed_message) }
def rle_test(self): filename = "testdata.txt" comp = Compressor(self.datapath + filename) origdata = open(self.datapath + filename, 'rb').read() print "Begining RLE Test!", comp.rle_encode() assert origdata == comp.rle_decode() print "..Passed"
def decompress(self, compresses_index, compression_type='var_byte'): self.index = dict() for term in compresses_index.keys(): self.index[term] = dict() for doc_id in compresses_index[term]: if compression_type == 'var_byte': self.index[term][doc_id] = Compressor.variable_byte_decode( compresses_index[term][doc_id]) else: self.index[term][doc_id] = Compressor.gamma_decode( compresses_index[term][doc_id])
def compress(self, compression_type='var_byte'): compressed_index = dict() for term in self.index.keys(): compressed_index[term] = dict() for doc_id in self.index[term]: if compression_type == 'var_byte': compressed_index[term][ doc_id] = Compressor.variable_byte_encode( self.index[term][doc_id]) else: compressed_index[term][doc_id] = Compressor.gamma_encode( self.index[term][doc_id]) return compressed_index
def __init__(self): self.compressor = Compressor() self.turbine = Turbine() self.initial_guess = self.__class__.DEFAULT_PARAMS.copy() self.cpc = 1004 # J/(kg K) self.cpt = 1225 # J/(kg K) self.T01 = 273.15 # K self.P01 = 101e3 # Pa self.R_c = (1 - 1 / self.compressor.gam) * self.cpc # from gam and cp self.R_t = (1 - 1 / self.turbine.gam) * self.cpt self.A8 = pi / 4 * 35e-3**2 # Measured: D8 = 45mm self.FHV = 43e6 #J/kg Fuel lower calorific value self.eta_combustion = 0.5
def __init__(self,V=1000.,cmp_eff=80.,trb_eff=90.,p=200000): # Air Side self.amb = Ambient() self.cmp = Compressor(eff=cmp_eff) self.tank = Tank(V=V,p=p) self.trb = Turbine(eff=trb_eff) self.m_dot_i = 0.0 self.m_dot_o = 0.0 # Heat Side # Create pandas DataFrame to store results self.df = pd.DataFrame(columns=variables)
async def get_compiled_file(self, session: aiohttp.ClientSession, target: BuildTarget, lock: asyncio.Lock, compressor: Compressor): print(f"Waiting for a compiled file from {self.address}") async with lock: data = aiohttp.FormData() archive_file = tempfile.NamedTemporaryFile(suffix='tar.xz') commands_file = tempfile.NamedTemporaryFile(suffix='.sh') target.substitute_for_absolute_paths('/') data.add_field('workdir', os.getcwd()) data.add_field('targets', ', '.join(target.target_files)) data.add_field('commands_file', commands_file.name) data.add_field('password', self.password) data.add_field('exact_lib_versions', 'true' if config.EXACT_LIB_VERSIONS else 'false') if self.libraries is None: self.libraries = [] response = await session.post(url=self.address + self.ALL_LIBRARIES_ENDPOINT) libraries = (await response.json()) for library in libraries: try: self.libraries.append(Library(library)) except: continue present_libraries = await self.get_present_libraries(target) async with lock: for library in present_libraries: await target.replace_in_commands( library, f'${{{library.split("/")[-1]}}}') await target.create_commands_file(commands_file.name) compressor.compress([ file for file in target.all_dependency_files if file not in present_libraries ] + [commands_file.name], archive_file.name) data.add_field('file', open(archive_file.name, 'rb'), filename=archive_file.name.split('/')[-1]) archive_file.close() commands_file.close() # Lock is released result = await self.get_archive(data, session) async with lock: await Compressor.extract_files(result)
def main(): p = argparse.ArgumentParser() p.add_argument('raw') p.add_argument('compressed', nargs='*') flags = p.parse_args() if ';' in flags.raw: flags.raw, flags.compressed = flags.raw.split(';') else: flags.compressed = flags.compressed[0] raw = cached_listdir_imgs(flags.raw, discard_shitty=False) compressed = cached_listdir_imgs(flags.compressed, discard_shitty=False) print(raw, compressed) print('Average bpp', np.mean([Compressor.bpp_from_compressed_file(p) for p in compressed.ps])) r = ResidualDataset(compressed, raw, 'diff', 256, True, output_device=pe.DEVICE) # for i in range(len(r)): # print(r[i]['residual'].unique()) # exit(1) d = DataLoader(r, batch_size=10, shuffle=False, num_workers=2) mi, ma = None, None for b in d: res = b['raw'] # print(res.unique()) if mi is None: mi = res.min() ma = res.max() else: mi = min(res.min(), mi) ma = max(res.max(), ma) print(mi, ma)
def get(self, fp, off=0, size=-1): """ Get object by `fingerprint` - Throws `ObjectNotFound` RADOS exception, if no object with that fingerprint exists - Return decompressed object - Return the whole object by default - If you specify `off` and `size` it is applied only after fetching the whole object """ self.log.debug("GET [%r]: %s:%s", fp, off, size) obj_size, _ = self.ioctx.stat(fp) compression_id = CAS._convert_meta( self.ioctx.get_xattr(fp, "cas.meta.compression")) decompressor = Compressor.select(compression_id) self.log.debug("GET [%r]: size %d compressed with %r", fp, obj_size, compression_id) compressed_data = self.ioctx.read(fp, obj_size, 0) data = decompressor.decompress(compressed_data) if size < 0: size = len(data) return data[off:off + size]
def _get_pils(self, idx): if self._cache and idx in self._cache: yield self._cache[idx] return compressed_p = self.compressed_images.ps[idx] bpp = Compressor.bpp_from_compressed_file(compressed_p) raw_p = self.compressed_to_raw[os.path.basename(compressed_p)] compressed, f1 = self._read_img(compressed_p) raw, f2 = self._read_img(raw_p) if not NO_ERRORS: assert compressed.size == raw.size, f'Error for {compressed_p}, {raw_p}; {compressed.size, raw.size}' if self._cache is not None and idx not in self._cache: with self._cache_lock: if idx not in self._cache: print('Caching', idx) compressed, raw = map(ResidualDataset._convert, (compressed, raw)) self._cache[idx] = compressed, raw, bpp yield compressed, raw, bpp f1.close() f2.close()
def get(self, fp, off=0, size=-1): """ Get object by `fingerprint` - Throws `ObjectNotFound` RADOS exception, if no object with that fingerprint exists - Return decompressed object - Return the whole object by default - If you specify `off` and `size` it is applied only after fetching the whole object """ self.log.debug("GET [%r]: %s:%s", fp, off, size) obj_size, _ = self.ioctx.stat(fp) compression_id = CAS._convert_meta(self.ioctx.get_xattr(fp, "cas.meta.compression")) decompressor = Compressor.select(compression_id) self.log.debug("GET [%r]: size %d compressed with %r", fp, obj_size, compression_id) compressed_data = self.ioctx.read(fp, obj_size, 0) data = decompressor.decompress(compressed_data) if size < 0: size = len(data) return data[off:off+size]
def make_bin_pkl(img_folder, num_bins, overwrite=False): if isinstance(img_folder, list): img_folder = img_folder[0] pkl_out = get_default_pkl_p(img_folder, num_bins) if os.path.isfile(pkl_out): return pkl_out assert 'train_oi' in img_folder, img_folder # currently not supported, bc min_size and discard_shitty flags ps = cached_listdir_imgs.cached_listdir_imgs(img_folder, min_size=None, discard_shitty=True).ps ps_bpsps = sorted( ((p, Compressor.bpp_from_compressed_file(p) / 3) for p in ps), key=lambda xy: xy[1]) # sort by bpsp bpsps = [bpsp for _, bpsp in ps_bpsps] # border b0 b1 ... bk k+1 borders # bin_idx 0 1 2 ... k k+1 => k+2 bins # # for N bins, we need N-1 borders # first border is after 1/NB-th of the data # NB + 1 so that we get NB-1 evenly spaced bins *within* the data bin_borders_x = np.linspace(0, len(bpsps) - 1, num_bins + 1, dtype=np.int) # throw away the boundaries bin_borders_x = bin_borders_x[1:-1] bin_borders_b = [bpsps[x] for x in bin_borders_x] with open(pkl_out, 'wb') as f: print('Saving', bin_borders_b, '\n->', pkl_out) pickle.dump(bin_borders_b, f) return pkl_out
def dump_tables(fontfile, output): font = TTFont(fontfile, lazy=True) dump_folder = output + '_tables' print('dump results in {0}'.format(dump_folder)) try: os.makedirs(dump_folder) except OSError as exception: if exception.errno != errno.EEXIST: raise header_dict = FontInfo.getInformation(fontfile, FontInfo.TAGS.keys()) bin_header = BaseHeaderPrepare.prepare(BaseFonter.BASE_VERSION, header_dict) print('Base header total size=', len(bin_header)) base_fonter = BaseFonter(fontfile) base_dump = dump_folder + '/base_dump' base_fonter.dump_base(base_dump) # OpenType tables. dump_file = open(base_dump, 'r+b') tables = font.reader.tables for name in font.reader.tables: table = tables[name] offset = table.offset length = table.length #print('{0}: offset={1}, length={2}'.format(name, offset, length)) table_file_name = dump_folder + '/' + name.replace('/', '_') table_file = open(table_file_name, 'w+b') dump_file.seek(offset) table_file.write(dump_file.read(length)) table_file.close() rle_table = RleFont(table_file_name) rle_table.encode() rle_table.write(table_file_name) compressor = Compressor(Compressor.GZIP_INPLACE_CMD) compressor.compress(table_file_name) print('{0}: offset={1:9d}\tlen={2:9d}\tcmp_len={3:9d}'.format( name, offset, length, os.path.getsize(table_file_name + '.gz'))) print('TODO(bstell) save and compress the CFF parts.') if 'CFF ' in font: dumpCFFTable(font) font.close()
def dump_tables(fontfile, output): font = TTFont(fontfile,lazy=True) dump_folder = output + '_tables' print('dump results in {0}'.format(dump_folder)) try: os.makedirs(dump_folder) except OSError as exception: if exception.errno != errno.EEXIST: raise header_dict = FontInfo.getInformation(fontfile, FontInfo.TAGS.keys()) bin_header = BaseHeaderPrepare.prepare(BaseFonter.BASE_VERSION, header_dict) print('Base header total size=',len(bin_header)) base_fonter = BaseFonter(fontfile) base_dump = dump_folder + '/base_dump' base_fonter.dump_base(base_dump) # OpenType tables. dump_file = open(base_dump,'r+b') tables = font.reader.tables for name in font.reader.tables: table = tables[name] offset = table.offset length = table.length #print('{0}: offset={1}, length={2}'.format(name, offset, length)) table_file_name = dump_folder + '/' + name.replace('/', '_') table_file = open(table_file_name,'w+b') dump_file.seek(offset); table_file.write(dump_file.read(length)) table_file.close() rle_table = RleFont(table_file_name) rle_table.encode() rle_table.write(table_file_name) compressor = Compressor(Compressor.GZIP_INPLACE_CMD) compressor.compress(table_file_name) print('{0}: offset={1:9d}\tlen={2:9d}\tcmp_len={3:9d}'.format(name, offset, length,os.path.getsize(table_file_name+'.gz'))) print('TODO(bstell) save and compress the CFF parts.') if 'CFF ' in font: dumpCFFTable(font) font.close()
def _filter_compressed_images(self, raw_images): """ Find all compressed images that do not have a raw image, and remove them! """ raw_images_fn_to_p = {os.path.splitext(os.path.basename(p))[0] for p in raw_images.ps} missing = set() for p in self.compressed_images.ps: if Compressor.filename_without_bpp(p) not in raw_images_fn_to_p: missing.add(p) if missing: print(f'*** Missing {len(missing)} files that are in compressed but not in raw.') self.compressed_images.ps = [p for p in self.compressed_images.ps if p not in missing]
def main(dataPath, datacollector, display): """Main function of the odb2collector. :param String dataPath: Directory where the data will be stored. :param DataCollector datacollector: DataCollector instance used. :param Display display: Display instance used to show messages. """ ## Numer of bytes stored loggedBytes = 0 ## Initiate the compressor gzip = Compressor() gzip.start() logEnding = CONFIGURATION["logending"] while True: message = ["Logged %skB" % (loggedBytes / 1024)] print "\n".join(message) display.write_message(message) datafile = "%s/%s.%s" % (dataPath, _getCurrentTime(), logEnding) loggedBytes += datacollector.write_data_log(datafile, nbrOfOBDFrames=1000, messagesPerTimestamp=20) print "Collected data log..." gzip.add_file_for_compression(datafile)
def test_generateHeader(self): myMap = { ord('r'): '01', ord('e'): '00', ord('j'): '11', ord('l'): '10', } chars = ['e', 'r', 'l', 'j'] payloadFormat = '001{0[0]}1{0[1]}01{0[2]}1{0[3]}' expected = self.getHeader(chars, payloadFormat) actual = Compressor(None, None).generateHeader(myMap) self.assertEqual(expected, actual)
def base_font(self): output = self.folder + '/base' baseFonter = BaseFonter(self.fontfile) baseFonter.base(output) compressor = Compressor(Compressor.LZMA_CMD) compressor.compress(output, output + '.xz') compressor = Compressor(Compressor.GZIP_CMD) compressor.compress(output, output + '.gz')
def __init__(self, ioctx, compression="no"): """ Initialize CAS object. Caller needs to provide a connected and initialized RADOS I/O context. CAS objects need their own, exclusive RADOS I/O Context, since they operate on objects in an extra namespace. On initialization you can also specify the compression algorithm for **new** objects. CAS never overwrites objects that already exists, but rather increments their reference count. Existing objects have metadata to select the right decompressor on `get`. """ self.ioctx = ioctx self.ioctx.set_namespace("CAS") self.compressor = Compressor.select(compression)
def test_generateHeader3(self): myMap = { ord('b'): '100', ord('p'): '01', ord('e'): '00', ord('o'): '101', ord('r'): '1110', ord('x'): '1111', ord('!'): '1101', ord(' '): '1100' } chars = ['e', 'p', 'b', 'o', ' ', '!', 'r', 'x'] payloadFormat = '001{0[0]}1{0[1]}001{0[2]}1{0[3]}001{0[4]}1{0[5]}01{0[6]}1{0[7]}' expected = self.getHeader(chars, payloadFormat) actual = Compressor(None, None).generateHeader(myMap) self.assertEqual(expected, actual)
def build(): logger = logging.create_logger(app) print(request.form) password = request.form['password'] if not is_password_acceptable(password): return make_response('', 400) # Extracting files tempdir = tempfile.TemporaryDirectory() print(request.files) f = request.files['file'] commands_file = request.form['commands_file'].strip('/') archive_file = tempfile.NamedTemporaryFile(suffix='.tar.xz', dir=tempdir.name) archive_filename = archive_file.name.split('/')[-1] print(archive_filename) f.save(archive_file.name) compressor = Compressor(tempdir.name) compressor.extract_files(archive_filename) new_root = f'{tempdir.name}/{archive_filename.split(".")[0]}/' archive_file.close() # Run commands print(new_root) command_runner = CommandRunner(new_root + request.form['workdir'].strip('/'), app.config['LIBRARIES']) output, code = command_runner.run_commands(new_root + commands_file, new_root, logger, request.form['exact_lib_versions'] == 'true') if code != 0: logger.debug(code) response = output, 400 else: # Sending files back target_files = [target.strip('/') for target in request.form['targets'].split(', ')] compressor.root_path = new_root output_file = tempfile.NamedTemporaryFile(suffix='.tar.xz') compressor.compress(target_files, output_file.name) response = send_file(output_file, mimetype='application/x-object') tempdir.cleanup() return response
def _compressed_to_raw(compressed_images: Images, raw_images: Images): """ Create mapping compressed_image -> raw_image. Can be many to one! """ raw_images_fn_to_p = {os.path.splitext(os.path.basename(p))[0]: p for p in raw_images.ps} compressed_to_raw_map = {} errors = [] for p in compressed_images.ps: try: fn = Compressor.filename_without_bpp(p) compressed_to_raw_map[os.path.basename(p)] = raw_images_fn_to_p[fn] except KeyError as e: errors.append(e) if len(errors) > 100: break continue if errors: raise ValueError(f'Missing >={len(errors)} keys:', errors[:10], f'\n{len(compressed_images.ps)} vs {len(raw_images.ps)}') return compressed_to_raw_map
async def main(): # Check the file with GNU make check_result = sp.Popen(f'make -n -f {config.MAKEFILE}', stderr=sp.PIPE, shell=True) output = check_result.communicate()[1] if check_result.returncode != 0: print("File seems to be of bad format") print(str(output)) exit(1) lines = open(config.MAKEFILE, 'r').readlines() p = Parser(lines) p.replace_all_variables() p.get_build_targets() rm = await RequestsManager.create(p.default_target, hosts_file, Compressor(), p) try: await rm.build_targets() except ExecutionError as e: print(e.commands_output, e.message)
from compressor import Compressor compressor = Compressor("static/test.png") print('') print("====================") print("= MATRICE =") print("====================") print('') matrice = compressor.get_matrice() for line in matrice: print(line) print('') print("====================") print("= ARRAY =") print("====================") print('') print(compressor.get_array())
class Engine(Turbomachine): def __init__(self): self.compressor = Compressor() self.turbine = Turbine() self.initial_guess = self.__class__.DEFAULT_PARAMS.copy() self.cpc = 1004 # J/(kg K) self.cpt = 1225 # J/(kg K) self.T01 = 273.15 # K self.P01 = 101e3 # Pa self.R_c = (1 - 1 / self.compressor.gam) * self.cpc # from gam and cp self.R_t = (1 - 1 / self.turbine.gam) * self.cpt self.A8 = pi / 4 * 35e-3**2 # Measured: D8 = 45mm self.FHV = 43e6 #J/kg Fuel lower calorific value self.eta_combustion = 0.5 DEFAULT_PARAMS = { 'P0_ratio_c': 1.2347241010782356, 'Mb_t': 0.27072466251896482, 'MFP4': 0.15310678698124691, 'MFP3': 0.14924987675288623, 'M_flight': 0, 'mdotf': 0.0019637316313999187, 'P0_ratio_t': 0.92680225375718472, 'MFP5': 0.15451547834023707, 'Mb_c': 0.44527731422329964, 'MFP': 0.14801196525452109, 'T0_ratio_t': 0.98261102832775471, 'T0_ratio_c': 1.0669702796449636, 'T04': 840.63888888888891 } N_FREE_PARAMS = 2 def implicit_map( self, M_flight, # flight mach number MFP, MFP3, Mb_c, T0_ratio_c, P0_ratio_c, # compressor mdotf, T04, # burner MFP4, MFP5, Mb_t, T0_ratio_t, P0_ratio_t # turbine ): """ This function implements burner, nozzle and spool dynamics and integrates it with turbine and compressor maps Variable balance: 12 variables - 11 equations -------- 2 free choices (e.g. M_flight and mdotf) """ # Expose constants A8 = self.A8 FHV = self.FHV eta_combustion = self.eta_combustion A5 = self.turbine.geom['A2'] gam_c = self.compressor.gam gam_t = self.turbine.gam cpc = self.cpc cpt = self.cpt T01 = self.T01 P01 = self.P01 R_c = self.R_c R_t = self.R_t ### Dimensionalize everything ### dim_params = self.dimensionalize( M_flight, # flight mach number MFP, MFP3, Mb_c, T0_ratio_c, P0_ratio_c, # compressor mdotf, T04, # burner MFP4, MFP5, Mb_t, T0_ratio_t, P0_ratio_t # turbine ) T02 = dim_params.T02 T03 = dim_params.T03 T05 = dim_params.T05 P02 = dim_params.P02 P03 = dim_params.P03 P04 = dim_params.P04 P05 = dim_params.P05 omega_c = dim_params.omega_c omega_t = dim_params.omega_t mdot2 = dim_params.mdot2 mdot3 = dim_params.mdot3 mdot4 = dim_params.mdot4 mdot5 = dim_params.mdot5 ### CONSTRAINTS ### # * Energy addition in ther burner res_T04 = T03 + mdotf * FHV * eta_combustion / (mdot3 * cpc) - T04 # * Spool has constant speed res_omega = omega_c - omega_t # * Conservation of energy in the spool res_energy = mdot2 * cpc * (T03 - T02) - mdot4 * cpt * (T04 - T05) # * Consevation of mass in the combustor res_mdot = mdot3 + mdotf - mdot4 # * Nozzle exit is either choked or at ambient pressure Pa = P01 / (1 + (gam_c - 1) / 2 * M_flight**2)**(gam_c / (gam_c - 1)) MFP8 = MFP5 * A5 / A8 res_MFP_nozzle = (P05 / Pa - 1) * MFP8 - (P05 / Pa - 1) * mach2mfp( min(1, (2 / (gam_t - 1) * ((P05 / Pa)**( (gam_t - 1) / gam_t) - 1))**0.5) if P05 / Pa > 1 else 0, gam_t) ### remove dimensions of residuals ### # References for removing dimensions of residuals mdotref = (T01 * R_c)**0.5 / (P01 * gam_c**0.5 * self.compressor.geom['A1']) h_ref = cpc * T01 omega_ref = (gam_c * R_c * T01)**0.5 / self.compressor.geom['D2'] #remove dimensions res_omega /= omega_ref res_energy /= h_ref * mdotref res_mdot /= mdotref res_T04 /= T04 ### COMPRESSOR MAP ### res_MFP_c, res_T0_ratio_c, res_P0_ratio_c = self.compressor.implicit_map( MFP, MFP3, Mb_c, T0_ratio_c, P0_ratio_c, tol=1e-13) ### TURBINE MAP ### res_MFP_t, res_T0_ratio_t, res_P0_ratio_t = self.turbine.implicit_map( MFP4, MFP5, Mb_t, T0_ratio_t, P0_ratio_t, tol=1e-13) return (res_omega, res_energy, res_mdot, res_MFP_nozzle, res_T04, res_MFP_c, res_T0_ratio_c, res_P0_ratio_c, res_MFP_t, res_T0_ratio_t, res_P0_ratio_t) def dimensionalize( self, M_flight, # flight mach number MFP, MFP3, Mb_c, T0_ratio_c, P0_ratio_c, # compressor mdotf, T04, # burner MFP4, MFP5, Mb_t, T0_ratio_t, P0_ratio_t # turbine ): gam_c = self.compressor.gam gam_t = self.turbine.gam T01 = self.T01 P01 = self.P01 R_c = self.R_c R_t = self.R_t ### Dimensionalize everything ### a01 = (gam_c * R_c * T01)**0.5 a04 = (gam_t * R_t * T04)**0.5 T02 = T01 T03 = T02 * T0_ratio_c T04 = T04 T05 = T04 * T0_ratio_t P02 = P01 P03 = P02 * P0_ratio_c P04 = P03 P05 = P03 * P0_ratio_t omega_c = Mb_c * a01 / (self.compressor.geom['D2'] / 2) omega_t = Mb_t * a04 / (self.turbine.geom['D1t'] / 2) mdot2 = MFP * self.compressor.geom['A1'] * P01 * gam_c**0.5 / ( T01 * R_c)**0.5 mdot3 = MFP3 * self.compressor.geom['A2'] * P03 * gam_c**0.5 / ( T03 * R_c)**0.5 mdot4 = MFP4 * self.turbine.geom['A1'] * P04 * gam_t**0.5 / (T04 * R_t)**0.5 mdot5 = MFP5 * self.turbine.geom['A2'] * P05 * gam_t**0.5 / (T05 * R_t)**0.5 #Nozzle MFP8 = min(MFP5 * self.turbine.geom['A2'] / self.A8, mach2mfp(1, gam_t)) M8 = mfp2mach(MFP8, gam_t) P8 = P05 * (1 + (gam_t - 1) / 2 * M8**2)**(-gam_t / (gam_t - 1)) T8 = T05 * (1 + (gam_t - 1) / 2 * M8**2)**(-1) dim_params = DimensionalParameters(T02=T02, T03=T03, T04=T04, T05=T05, T8=T8, P02=P02, P03=P03, P04=P04, P05=P05, P8=P8, omega_c=omega_c, omega_t=omega_t, mdot2=mdot2, mdot3=mdot3, mdot4=mdot4, mdot5=mdot5) return dim_params def working_line(self, T04_grid): wline = [] params = self.initial_guess for T04 in T04_grid: sol = self.general_explicit_map({ 'M_flight': 0, 'T04': T04 }, params) params = sol.params wline.append(params) return wline def thrust(self, **params): gam_t = self.turbine.gam R_t = self.R_t MFP8 = min(params['MFP5'] * self.turbine.geom['A2'] / self.A8, mach2mfp(1, gam_t)) mdot = params['MFP5'] * self.turbine.geom['A2'] * params[ 'P05'] * gam_t**0.5 / (params['T05'] * R_t)**0.5 M8 = mfp2mach(MFP8, gam_t) P8 = params['P05'] * (1 + (gam_t - 1) / 2 * M8**2)**(-gam_t / (gam_t - 1)) mdot5 = params['MFP5'] * self.turbine.geom['A2'] * params[ 'P05'] * gam_t**0.5 / (params['T05'] * R_t)**0.5 T8 = params['T05'] * (1 + (gam_t - 1) / 2 * M8**2)**(-1) return mdot * M8 * (gam_t * R_t * T8)**0.5 + ( P8 - self.P01 / (1 + (gam_t - 1) * params['M_flight']**2)**0.5) * self.A8 def ode_fun(self, t, y): P4, mdot, omega = y self.compressor.explicit_map(mdot, omega) T04 = self.throtle(t, mdot, T03) self.turbine.explicit_map(P4, omega) P4_dot = a01**2 * (mdot - mdott) mdotdot = A1 / Lc * (P3 - P4) omega_dot = 1 / J * torque_t - torque_c return P4_dot, mdotdot, omega_dot
return total_work / self.eta else: return self.compressor_in.work_done / self.eta except AttributeError as e: raise e if __name__ == '__main__': from inlet import Inlet from fan import Fan from bypass import Bypass from compressor import Compressor ambient_conditions = FlowCondition(corrected_mass_flow=1400., mach=0.8, t_static=216, p_static=22632, station_number='1', medium='air') inlet = Inlet(ambient=ambient_conditions, eta=0.98) fan = Fan(inflow=inlet.outflow, eta=0.92, pressure_ratio=1.6, station_number='21') bypass = Bypass(inflow=fan.outflow, bypass_ratio=8.) lpc = Compressor(inflow=bypass.outflow_core, eta=0.9, pressure_ratio=1.4, station_number='25') hpc = Compressor(inflow=lpc.outflow, eta=0.9, pressure_ratio=19, station_number='3') lp_spool = Spool(compressor_in=(fan, lpc), eta=0.99) print(lp_spool.work_required) print(lpc.work_done) print(hpc.inflow.mass_flow) print(hpc.inflow.t_total) print(hpc.p_total) print(hpc.t_total) print(hpc.work_done) # print(obj.p_total) # print(obj.t_total)
def main(): try: init_options() except OptionsError as e: sys.stderr.write("Error: {0}\n\n".format(e)) sys.stderr.write(usage()) sys.stderr.write("\n"); sys.stderr.flush() sys.exit(os.EX_CONFIG) if options.main.generate_config: print(generate_sample_config()) # XXX: this doesn't yet work properly because of groper sys.exit() conf_root = os.path.dirname(os.path.abspath(options.main.config)) facility_db = FacilityDB() try: facility_db.load_config(normalize_path(options.main.facilities_config, conf_root)) except (IOError) as e: sys.stderr.write("Error reading {0}: {1}\n".format(options.main.facilities_config, e)) sys.stderr.flush() sys.exit(os.EX_CONFIG) except (FacilityError, configparser.Error) as e: sys.stderr.write("{0} contains errors:\n\n".format(options.main.facilities_config)) if hasattr(e, 'lineno'): e = 'Error on line {0}: {1}'.format(e.lineno, e.message.split("\n")[0].strip()) sys.stderr.write("{0}\n\n".format(e)) sys.stderr.write("Exiting now.\n") sys.stderr.flush() sys.exit(os.EX_CONFIG) if options.main.check_config: sys.exit() # We are just checking the config file, so exit here. cache_config_checksum() create_dirs() if options.main.daemon: daemonize() if options.main.user: drop_privileges(options.main.user) if options.main.pidfile: write_pid(options.main.pidfile) atexit.register(exit_handler) setup_logging() try: logging.getLogger().info("Starting loghogd.") compressor = Compressor() compressor.find_uncompressed(options.main.logdir, r'.+\.log\..+') writer = Writer(facility_db, compressor, options.main.logdir) processor = Processor(facility_db, writer) server = Server(processor.on_message, conf_root) signal_handler = make_shutdown_handler(server, writer, compressor) signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGHUP, make_reload_handler(facility_db, writer)) except Exception as e: logging.getLogger().error(e) logging.getLogger().error('Exiting abnormally due to an error at startup.') sys.exit(os.EX_CONFIG) try: compressor.start() server.run() except Exception as e: logging.getLogger().exception(e) logging.getLogger().error('Exiting abnormally due to an error at runtime.') shutdown(None, server, writer, compressor) sys.exit(os.EX_SOFTWARE) logging.getLogger().info('Shutdown complete. Exiting.')
from compressor import Compressor from decompressor import Decompressor inputFileName = 'debug.txt' compressedFileName = 'compress.zip' decompressedFileName = 'uncompress.txt' Compressor(inputFileName, compressedFileName).run() Decompressor(compressedFileName, decompressedFileName).run()
from server import Server from parse import Parser from proxy_server import Proxy print("SSSSSSSSSSSSSSSSSSSSSSSSS") # initiate server object. server = Server() print("PPPPPPPPPPPPPPPPPPPPPPPPPPPP") # initiate parser object. parsers = Parser() print("SSSSSSSSCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC") # initiate scraper object. scrape = Scraper() print("CoCoCoCoCoCoCoCoCoCoCo") # initiate compressor object. compressor = Compressor() print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@") # creates a connection pool to Server database. server_mongodb = MongoDB(db_name='Server') print("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") # creates a connection pool to Compressor database. compressor_mongodb = MongoDB(db_name='Compressor') print("YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY") # counter and document id variables initial values. crawler_doc_id = 0 count_crawler = 0 compressor_doc_id = 0 mongodb_doc_id = 0 #doc_id = 0
def main(): try: init_options() except OptionsError as e: sys.stderr.write("Error: {0}\n\n".format(e)) sys.stderr.write(usage()) sys.stderr.write("\n") sys.stderr.flush() sys.exit(os.EX_CONFIG) if options.main.generate_config: print(generate_sample_config() ) # XXX: this doesn't yet work properly because of groper sys.exit() conf_root = os.path.dirname(os.path.abspath(options.main.config)) facility_db = FacilityDB() try: facility_db.load_config( normalize_path(options.main.facilities_config, conf_root)) except (IOError) as e: sys.stderr.write("Error reading {0}: {1}\n".format( options.main.facilities_config, e)) sys.stderr.flush() sys.exit(os.EX_CONFIG) except (FacilityError, configparser.Error) as e: sys.stderr.write("{0} contains errors:\n\n".format( options.main.facilities_config)) if hasattr(e, 'lineno'): e = 'Error on line {0}: {1}'.format( e.lineno, e.message.split("\n")[0].strip()) sys.stderr.write("{0}\n\n".format(e)) sys.stderr.write("Exiting now.\n") sys.stderr.flush() sys.exit(os.EX_CONFIG) if options.main.check_config: sys.exit() # We are just checking the config file, so exit here. cache_config_checksum() create_dirs() if options.main.daemon: daemonize() if options.main.user: drop_privileges(options.main.user) if options.main.pidfile: write_pid(options.main.pidfile) atexit.register(exit_handler) setup_logging() try: logging.getLogger().info("Starting loghogd.") compressor = Compressor() compressor.find_uncompressed(options.main.logdir, r'.+\.log\..+') writer = Writer(facility_db, compressor, options.main.logdir) processor = Processor(facility_db, writer) server = Server(processor.on_message, conf_root) signal_handler = make_shutdown_handler(server, writer, compressor) signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGHUP, make_reload_handler(facility_db, writer)) except Exception as e: logging.getLogger().error(e) logging.getLogger().error( 'Exiting abnormally due to an error at startup.') sys.exit(os.EX_CONFIG) try: compressor.start() server.run() except Exception as e: logging.getLogger().exception(e) logging.getLogger().error( 'Exiting abnormally due to an error at runtime.') shutdown(None, server, writer, compressor) sys.exit(os.EX_SOFTWARE) logging.getLogger().info('Shutdown complete. Exiting.')
def learn_sparse_embedding(session, model, verbose=False, output_frequency=500): embedding_param = session.run(model.embedding) embedding_basis = embedding_param[:model. basis_size, :] # Choose first model.basis_size rows as basis embedding_sp_ids_val = np.zeros(model.vocab_size_in * model.sparsity, dtype=np_index_data_type()) embedding_sp_weights_val = np.zeros(model.vocab_size_in * model.sparsity) # For basis vectors, they have one-hot expressions for i in range(model.basis_size): random_basis = {i: True} j = 1 # j: number of elements in random_basis while j < model.sparsity: r = random.randint(0, model.basis_size - 1) if (r not in random_basis): random_basis[r] = True j += 1 del random_basis[i] # 1 true basis, model.sparsity - 1 paddings indices = np.r_[ np.array([i], dtype=np_index_data_type()), np.array(random_basis.keys(), dtype=np_index_data_type())] embedding_sp_ids_val[i * model.sparsity:(i + 1) * model.sparsity] = indices # other weights are zero by the definition of `softmax_sp_weights_val` embedding_sp_weights_val[i * model.sparsity] = 1.0 t1 = time.time() # print("max non zero entry: ", model.sparsity) # columns are bases. Need to transpose. compressor = Compressor(bases=embedding_basis.T, max_non_zero_entry=model.sparsity) pool = multiprocessing.Pool(processes=FLAGS.cpu_count) parallel_params = [(compressor, embedding_param[i, :]) for i in range(model.basis_size, model.vocab_size_in)] results = pool.map(fit_wrapper, parallel_params) for i_, result in enumerate(results): indices, values = result i = i_ + model.basis_size embedding_sp_ids_val[i * model.sparsity:(i + 1) * model.sparsity] = indices embedding_sp_weights_val[i * model.sparsity:(i + 1) * model.sparsity] = values t2 = time.time() print("Parallel time: ", t2 - t1) finetune_save_path = os.path.join(FLAGS.save_path, "finetune-" + FLAGS.model_config) if not os.path.isdir(finetune_save_path): os.mkdir(finetune_save_path) sparse_parameters_path = finetune_save_path np.save(os.path.join(sparse_parameters_path, "embedding_sp_ids_val.npy"), embedding_sp_ids_val) np.save( os.path.join(sparse_parameters_path, "embedding_sp_weights_val.npy"), embedding_sp_weights_val) model.assign_sparse_embedding_params(session, embedding_basis) return embedding_basis, embedding_sp_ids_val, embedding_sp_weights_val
def compress_files(self, files): compressor = Compressor(files, self.runner.results_path, debug=self.debug) return compressor.zip()
def test_processFile4(self): lastByte = '1000001' expected = [130, 1] actual = Compressor(None, None).processEOF(lastByte, []) self.assertEqual(expected, actual)
import matplotlib.pyplot as plt from compressor import Compressor fig, ax = plt.subplots(1, 1, figsize=(4, 3)) c = Compressor() c.plot_map(ax, samples=144) plt.savefig('compressor_map.pdf')
import matplotlib.pyplot as plt plt.rc('text', usetex=True) plt.rc('font', family='serif') from compressor import Compressor plt.figure(figsize=(6,8),dpi=150) Compressor().plot_map(plt.gca(), samples=100, plot=True) plt.savefig('map.pdf', dpi=300) plt.show()