def testLargeFileException(self): for f in (TESTFN2, TemporaryFile(), StringIO()): self.largeFileExceptionTest(f, zipfile.ZIP_STORED) self.largeFileExceptionTest2(f, zipfile.ZIP_STORED)
def fftype(mol, rtfFile=None, prmFile=None, method='GAFF2', acCharges=None, tmpDir=None, netcharge=None): """ Assing atom types and force field parameters for a given molecule. Additionally, atom masses and improper dihedral are set. Optionally, atom charges can be set if `acCharges` is set (see below). The assignment can be done: 1. For CHARMM CGenFF_2b6 with MATCH (method = 'CGenFF_2b6'); 2. For AMBER GAFF with antechamber (method = 'GAFF'); 3. For AMBER GAFF2 with antechamber (method = 'GAFF2'); Parameters ---------- mol : Molecule Molecule to use for the assignment rtfFile : str Path to a RTF file from which to read the topology prmFile : str Path to a PRM file from which to read the parameters method : str Atomtyping assignment method. Use :func:`fftype.listFftypemethods <htmd.parameterization.fftype.listFftypemethods>` to get a list of available methods. Default: :func:`fftype.defaultFftypemethod <htmd.parameterization.fftype.defaultFftypemethod>` acCharges : str Optionally assign charges with antechamber. Check `antechamber -L` for available options. Note: only works for GAFF and GAFF2. tmpDir: str Directory for temporary files. If None, a directory is created and deleted automatically. netcharge : float The net charge of the molecule. Returns ------- prm : :class:`ParameterSet <parmed.parameters.ParameterSet>` object Returns a parmed ParameterSet object with the parameters. mol : :class:`Molecule <moleculekit.molecule.Molecule>` object The modified Molecule object with the matching atom types for the ParameterSet """ import parmed if method not in fftypemethods: raise ValueError('Invalid method {}. Available methods {}'.format( method, ','.join(fftypemethods))) if method == 'CGenFF_2b6' and acCharges: raise ValueError('acCharges') if netcharge is None: netcharge = int(round(np.sum(mol.charge))) logger.warning( 'Molecular charge is set to {} by adding up the atomic charges'. format(netcharge)) if rtfFile and prmFile: from htmd.parameterization.readers import readRTF logger.info('Reading FF parameters from {} and {}'.format( rtfFile, prmFile)) prm = parmed.charmm.CharmmParameterSet(rtfFile, prmFile) names, elements, atomtypes, charges, masses, impropers = readRTF( rtfFile) else: logger.info('Assigning atom types with {}'.format(method)) renamed_mol = _canonicalizeAtomNames(mol) # Create a temporary directory with TemporaryDirectory() as tmpdir: # HACK to keep the files tmpdir = tmpdir if tmpDir is None else tmpDir logger.debug('Temporary directory: {}'.format(tmpdir)) if method in ('GAFF', 'GAFF2'): from moleculekit.molecule import Molecule from htmd.parameterization.readers import readPREPI, readFRCMOD # Write the molecule to a file renamed_mol.write(os.path.join(tmpdir, 'mol.mol2')) atomtype = method.lower() # Set arguments cmd = [ 'antechamber', '-at', atomtype, '-nc', str(netcharge), '-fi', 'mol2', '-i', 'mol.mol2', '-fo', 'prepi', '-o', 'mol.prepi' ] if acCharges is not None: cmd += ['-c', acCharges] # Run antechamber with TemporaryFile() as stream: if subprocess.call( cmd, cwd=tmpdir, stdout=stream, stderr=stream) != 0: raise RuntimeError('"antechamber" failed') stream.seek(0) for line in stream.readlines(): logger.debug(line) # Set arguments cmd = [ 'parmchk2', '-f', 'prepi', '-s', atomtype, '-i', 'mol.prepi', '-o', 'mol.frcmod', '-a', 'Y' ] # Run parmchk2 with TemporaryFile() as stream: if subprocess.call( cmd, cwd=tmpdir, stdout=stream, stderr=stream) != 0: raise RuntimeError('"parmchk2" failed') stream.seek(0) for line in stream.readlines(): logger.debug(line) # Check if antechamber did changes in atom names (and suggest the user to fix the names) acmol = Molecule(os.path.join(tmpdir, 'NEWPDB.PDB'), type='pdb') acmol.name = np.array([n.upper() for n in acmol.name]).astype(np.object) changed_mol_acmol = np.setdiff1d(renamed_mol.name, acmol.name) changed_acmol_mol = np.setdiff1d(acmol.name, renamed_mol.name) if len(changed_mol_acmol) != 0 or len(changed_acmol_mol) != 0: raise RuntimeError( 'Initial atom names {} were changed by antechamber to {}. ' 'This probably means that the start of the atom name does not match ' 'element symbol. ' 'Please check the molecule.' ''.format(','.join(changed_mol_acmol), ','.join(changed_acmol_mol))) # Read the results prm = parmed.amber.AmberParameterSet( os.path.join(tmpdir, 'mol.frcmod')) names, atomtypes, charges, impropers = readPREPI( renamed_mol, os.path.join(tmpdir, 'mol.prepi')) masses, elements = readFRCMOD( atomtypes, os.path.join(tmpdir, 'mol.frcmod')) elif method == 'CGenFF_2b6': from htmd.parameterization.readers import readRTF # Write the molecule to a file renamed_mol.write(os.path.join(tmpdir, 'mol.pdb')) # Set arguments cmd = [ 'match-typer', '-charge', str(netcharge), '-forcefield', 'top_all36_cgenff_new', 'mol.pdb' ] # Run match-type with TemporaryFile() as stream: if subprocess.call( cmd, cwd=tmpdir, stdout=stream, stderr=stream) != 0: raise RuntimeError('"match-typer" failed') stream.seek(0) for line in stream.readlines(): logger.debug(line) prm = parmed.charmm.CharmmParameterSet( os.path.join(tmpdir, 'mol.rtf'), os.path.join(tmpdir, 'mol.prm')) names, elements, atomtypes, charges, masses, impropers = readRTF( os.path.join(tmpdir, 'mol.rtf')) else: raise ValueError('Invalid method {}'.format(method)) assert np.all(renamed_mol.name == names) assert np.all(mol.element == elements) mol = mol.copy() mol.atomtype = atomtypes mol.masses = masses mol.impropers = impropers if acCharges is not None: mol.charge = charges return prm, mol
past_weeks_4 = np.append(speed4.T, speed4.T, axis=1) past_weeks = np.append(past_weeks_1, past_weeks_2, axis=0) past_weeks = np.append(past_weeks, past_weeks_3, axis=0) past_weeks = np.append(past_weeks, past_weeks_4, axis=0) patterns = past_weeks f_particles = np.random.choice([0, 1], size=(8, 100)) fantasy_particles = np.append(f_particles, f_particles, axis=1) iterations = 100 ################################################################# #Learn the distribution learn(patterns, fantasy_particles, modelParameters, units, numConnections, connections_index, iterations) elapsed_time = time.time() - start_time ################################################################ #Test the model's prediction start_time = time.time() recovered = recall(speed_test[:, 1].T, units, modelParameters) elapsed_time_testing = time.time() - start_time print("Recovered: ", recovered[42:45]) print("Actual Speed: ", speed_test[42:45, 1]) print elapsed_time, elapsed_time_testing from tempfile import TemporaryFile recoveredSpeed = TemporaryFile() np.savez('recoveredSpeed_fA_2', recovered=recovered, ActualSpeed=speed_test[:, 1])
def run(self, data, store, signal, context, **kwargs): """ The main run method of the Python task. Args: data (MultiTaskData): The data object that has been passed from the predecessor task. store (DataStoreDocument): The persistent data store object that allows the task to store data for access across the current workflow run. signal (TaskSignal): The signal object for tasks. It wraps the construction and sending of signals into easy to use methods. context (TaskContext): The context in which the tasks runs. Returns: Action: An Action object containing the data that should be passed on to the next task and optionally a list of successor tasks that should be executed. """ params = self.params.eval(data, store, exclude=['command']) capture_stdout = self._callback_stdout is not None or params.capture_stdout capture_stderr = self._callback_stderr is not None or params.capture_stderr stdout_file = TemporaryFile() if params.capture_stdout else None stderr_file = TemporaryFile() if params.capture_stderr else None stdout = PIPE if capture_stdout else None stderr = PIPE if capture_stderr else None # change the user or group under which the process should run if params.user is not None or params.group is not None: pre_exec = self._run_as(params.user, params.group) else: pre_exec = None # call the command proc = Popen(self.params.eval_single('command', data, store), cwd=params.cwd, shell=True, env=params.env, preexec_fn=pre_exec, stdout=stdout, stderr=stderr, stdin=PIPE if params.stdin is not None else None) # if input is available, send it to the process if params.stdin is not None: proc.stdin.write(params.stdin.encode(sys.getfilesystemencoding())) # send a notification that the process has been started try: if self._callback_process is not None: self._callback_process(proc.pid, data, store, signal, context) except (StopTask, AbortWorkflow): proc.terminate() raise # send the output handling to a thread if capture_stdout or capture_stderr: output_reader = BashTaskOutputReader(proc, stdout_file, stderr_file, self._callback_stdout, self._callback_stderr, params.refresh_time, data, store, signal, context) output_reader.start() else: output_reader = None # wait for the process to complete and watch for a stop signal while proc.poll() is None or\ (output_reader is not None and output_reader.is_alive()): sleep(params.refresh_time) if signal.is_stopped: proc.terminate() if output_reader is not None: output_reader.join() data = output_reader.data # if a stop or abort exception was raised, stop the bash process and re-raise if output_reader.exc_obj is not None: if proc.poll() is None: proc.terminate() raise output_reader.exc_obj # send a notification that the process has completed if self._callback_end is not None: if stdout_file is not None: stdout_file.seek(0) if stderr_file is not None: stderr_file.seek(0) self._callback_end(proc.returncode, stdout_file, stderr_file, data, store, signal, context) if stdout_file is not None: stdout_file.close() if stderr_file is not None: stderr_file.close() return Action(data)
#!/usr/bin/env python3 # encoding=utf-8 from tempfile import TemporaryFile, NamedTemporaryFile, TemporaryDirectory with TemporaryFile('w+t') as f: f.write('heyheyhey') f.seek(0) data = f.read() print(data) with NamedTemporaryFile('w+t') as f: f.write('I\'m NamedTemporaryFile') f.seek(0) data = f.read() print(data) print(f.name) with TemporaryDirectory() as d: print('dirname: ' + d)
#!/usr/bin/env python # -*- coding:utf-8 -*- # __author__ = 'liao gao xiang' import tempfile from tempfile import TemporaryFile, TemporaryDirectory, NamedTemporaryFile # 你需要在程序执行时创建一个临时文件或目录,并希望使用完之后可以自动销毁掉。 with TemporaryFile('w+t', encoding='utf-8') as f: f.write('liaogx\n') f.write('Python') f.seek(0) print(f.read()) """ TemporaryFile() 的第一个参数是文件模式,通常来讲文本模式使用 w+t ,二进 制模式使用 w+b 。这个模式同时支持读和写操作,在这里是很有用的,因为当你关闭 文件去改变模式的时候,文件实际上已经不存在了 """ # 命名的临时文件,且关闭文件时不删除 with NamedTemporaryFile('w+t', encoding='utf-8', delete=False) as f: f.write(f.name) # 创建临时目录 with TemporaryDirectory() as dirname: print('dirname is: ', dirname) # 通常来讲,临时文件在系统默认的位置被创建,比如 /var/tmp 或类似的地方。为 # 了获取真实的位置,可以使用 tempfile.gettempdir() 函数
def check_example(package, filename): example_dir = path.join(examples, package) expected_dir = path.join(expected, package) expected_base = path.join(expected_dir, path.splitext(filename)[0]) try: with TempDirectory() as actual: # copy files to the directory copy(path.join(example_dir, filename), actual.path) for pattern in ('*.xls', '*.bmp'): for fixture in glob(path.join(example_dir, pattern)): copy(fixture, actual.path) os.chdir(actual.path) output = TemporaryFile('w+') # run the example before_listing = set(os.listdir(actual.path)) call([runner, filename], stdout=output, stderr=STDOUT) after_listing = set(os.listdir(actual.path)) # check the console output output.seek(0) actual_output = output.read().strip().replace('\r', '') for re, rp in sub_res: actual_output = re.sub(rp, actual_output) expected_path = expected_base + '.txt' if not path.exists(expected_path): expected_output = '' else: expected_output = open(expected_path).read().strip().replace( '\r', '') compare(expected_output, actual_output) # check the files created created = after_listing.difference(before_listing) expected_names = set() if os.path.exists(expected_base): expected_names = set(os.listdir(expected_base)) for name in created: with open(path.join(actual.path, name), 'rb') as af: actual_data = af.read() if name in expected_names: expected_path = path.join(expected_base, name) expected_data = open(expected_path, 'rb').read() expected_names.remove(name) if actual_data != expected_data: if environ.get('REPLACE_EXAMPLES'): with open(expected_path, 'wb') as new_expected: new_expected.write(actual_data) compare( get_biff_records(expected_data), get_biff_records(actual_data), ) else: raise AssertionError("unexpected output: %s" % name) for name in expected_names: if name != '.svn': print created raise AssertionError("expected output missing: %s" % name) finally: os.chdir(initial)
def persist_lines_job(project_id, dataset_id, lines=None): state = None schemas = {} key_properties = {} tables = {} rows = {} errors = {} bigquery_client = bigquery.Client(project=project_id) # try: # dataset = bigquery_client.create_dataset(Dataset(dataset_ref)) or Dataset(dataset_ref) # except exceptions.Conflict: # pass for line in lines: try: msg = singer.parse_message(line) except json.decoder.JSONDecodeError: logger.error("Unable to parse:\n{}".format(line)) raise if isinstance(msg, singer.RecordMessage): if msg.stream not in schemas: raise Exception("A record for stream {} was encountered before a corresponding schema".format(msg.stream)) schema = schemas[msg.stream] validate(msg.record, schema) dat = bytes(str(json.loads(json.dumps(msg.record), object_pairs_hook=clear_dict_hook)) + '\n', 'UTF-8') rows[msg.stream].write(dat) #rows[msg.stream].write(bytes(str(msg.record) + '\n', 'UTF-8')) state = None elif isinstance(msg, singer.StateMessage): logger.debug('Setting state to {}'.format(msg.value)) state = msg.value elif isinstance(msg, singer.SchemaMessage): table = msg.stream schemas[table] = msg.schema key_properties[table] = msg.key_properties #tables[table] = bigquery.Table(dataset.table(table), schema=build_schema(schemas[table])) rows[table] = TemporaryFile(mode='w+b') errors[table] = None # try: # tables[table] = bigquery_client.create_table(tables[table]) # except exceptions.Conflict: # pass else: raise Exception("Unrecognized message {}".format(msg)) for table in rows.keys(): table_ref = bigquery_client.dataset(dataset_id).table(table) SCHEMA = build_schema(schemas[table]) load_config = LoadJobConfig() load_config.schema = SCHEMA load_config.source_format = SourceFormat.NEWLINE_DELIMITED_JSON rows[table].seek(0) logger.info("loading {} to Bigquery.\n".format(table)) load_job = bigquery_client.load_table_from_file( rows[table], table_ref, job_config=load_config) logger.info("loading job {}".format(load_job.job_id)) logger.info(load_job.result()) # for table in errors.keys(): # if not errors[table]: # print('Loaded {} row(s) into {}:{}'.format(rows[table], dataset_id, table), tables[table].path) # else: # print('Errors:', errors[table], sep=" ") return state
def get_torch_object_bytes(obj): with TemporaryFile() as f: torch_save(obj, f) f.seek(0) b = f.read() return b
def telegramImgSend(url): with TemporaryFile() as f: f.write(requests.get(url).content) f.seek(0) bot.sendPhoto(TELEGRAM_CHAT_ID, f)
def start_services(self): self.server = subprocess.Popen(['python', abspath(__file__)], stdout=TemporaryFile(), stderr=subprocess.STDOUT)
def testRandomOpenStored(self): for f in (TESTFN2, TemporaryFile(), StringIO()): self.zipRandomOpenTest(f, zipfile.ZIP_STORED)
def testWriteNonPyfile(self): zipfp = zipfile.PyZipFile(TemporaryFile(), "w") file(TESTFN, 'w').write('most definitely not a python file') self.assertRaises(RuntimeError, zipfp.writepy, TESTFN) os.remove(TESTFN)
def testDeflated(self): for f in (TESTFN2, TemporaryFile(), StringIO()): self.zipTest(f, zipfile.ZIP_DEFLATED)
from gtts import gTTS import os from pygame import mixer from tempfile import TemporaryFile from io import BytesIO import time tts = gTTS(text='Hello world.', lang='en') #tts.save("hello.mp3") #os.system("mpg321 hello.mp3") print("init mixer") mixer.init() #mp3_fp = BytesIO() mp3_fp = TemporaryFile() print("Write to bytes like") tts.write_to_fp(mp3_fp) mp3_fp.seek(0) print("load file in mixer") mixer.music.load(mp3_fp) print("play sound") mixer.music.play() while mixer.music.get_busy(): time.sleep(0.1) print("Done")
def load_torch_object_bytes(b): with TemporaryFile() as f: f.write(b) f.seek(0) obj = torch_load(f) return obj
def draw(self, context, t): mode = self.get_string("mode", t, "clamp") img_ease = self.get_bool("img_ease", t, True) img_speed = self.get_number("img_speed", t, 1) if img_ease: img = self.get_image("image_surfaces", self.interpolate(self.no_interp_time * img_speed), None, mode) else: img = self.get_image("image_surfaces", self.no_interp_time * img_speed, None, mode) x = self.get_number("x", t, 100) y = self.get_number("y", t, 100) w = self.get_number("w", t, None) h = self.get_number("h", t, None) scale_x = self.get_number("scale_x", t, 1) scale_y = self.get_number("scale_y", t, 1) alpha = self.get_number("alpha", t, 1) rotation = rad(self.get_number("rotation", t, 0)) tint = self.get_color("tint", t, None) tint_op = self.get_cairo_constant("operator", "tint_op", t, cairo.OPERATOR_HSL_COLOR) if not img: return if w is None: w = img.get_width() if h is None: h = img.get_height() if tint is not None: # necessary to prevent reuse f = TemporaryFile() img.write_to_png(f) f.seek(0) _img = cairo.ImageSurface.create_from_png(f) b = cairo.Context(_img) # create alpha mask mask = cairo.ImageSurface(cairo.FORMAT_A8, img.get_width(), img.get_height()) maskctx = cairo.Context(mask) maskctx.set_source_surface(img) maskctx.paint() # paint b.set_operator(tint_op) b.set_source_rgba(*tint) b.mask(cairo.SurfacePattern(mask)) else: _img = img context.translate(x, y) context.scale(scale_x, scale_y) context.rotate(rotation) if self.get_bool("centered", t, True): context.translate(-0.5 * w, -0.5 * h) context.scale(w / _img.get_width(), h / _img.get_height()) context.set_source_surface(_img) context.paint_with_alpha(alpha)
def prepare(self, n_samples=None, n_features=None, dtype=None, X=None): """ Init estimator attributes based on input shape and type. Parameters ---------- n_samples: int, n_features: int, dtype: dtype in np.float32, np.float64 to use in the estimator. Override X.dtype if provided X: ndarray, shape (> n_components, n_features) Array to use to determine shape and types, and init dictionary if provided Returns ------- self """ if X is not None: X = check_array(X, order='C', dtype=[np.float32, np.float64]) if dtype is None: dtype = X.dtype # Transpose to fit usual column streaming this_n_samples = X.shape[0] if n_samples is None: n_samples = this_n_samples if n_features is None: n_features = X.shape[1] else: if n_features != X.shape[1]: raise ValueError('n_features and X does not match') else: if n_features is None or n_samples is None: raise ValueError('Either provide' 'shape or data to function prepare.') if dtype is None: dtype = np.float64 elif dtype not in [np.float32, np.float64]: return ValueError('dtype should be float32 or float64') # Regression statistics if self.G_agg == 'average': with TemporaryFile() as self.G_average_mmap_: self.G_average_mmap_ = TemporaryFile() self.G_average_ = np.memmap(self.G_average_mmap_, mode='w+', shape=(n_samples, self.n_components, self.n_components), dtype=dtype) atexit.register(self._exit) self.Dx_average_ = np.zeros((n_samples, self.n_components), dtype=dtype) # Dictionary statistics self.C_ = np.zeros((self.n_components, self.n_components), dtype=dtype) self.B_ = np.zeros((self.n_components, n_features), dtype=dtype) self.gradient_ = np.zeros((self.n_components, n_features), dtype=dtype, order='F') self.random_state = check_random_state(self.random_state) if X is None: self.components_ = np.empty((self.n_components, n_features), dtype=dtype) self.components_[:, :] = self.random_state.randn( self.n_components, n_features) else: random_idx = self.random_state.permutation( this_n_samples)[:self.n_components] self.components_ = check_array(X[random_idx], dtype=dtype.type, copy=True) if self.comp_pos: self.components_[self.components_ <= 0] = \ - self.components_[self.components_ <= 0] for i in range(self.n_components): enet_scale(self.components_[i], l1_ratio=self.comp_l1_ratio, radius=1) self.code_ = np.ones((n_samples, self.n_components), dtype=dtype) self.labels_ = np.arange(n_samples) self.comp_norm_ = np.zeros(self.n_components, dtype=dtype) if self.G_agg == 'full': self.G_ = self.components_.dot(self.components_.T) self.n_iter_ = 0 self.sample_n_iter_ = np.zeros(n_samples, dtype='int') self.random_state = check_random_state(self.random_state) random_seed = self.random_state.randint(MAX_INT) self.feature_sampler_ = Sampler(n_features, self.rand_size, self.replacement, random_seed) if self.verbose: log_lim = log(n_samples * self.n_epochs / self.batch_size, 10) self.verbose_iter_ = (np.logspace( 0, log_lim, self.verbose, base=10) - 1) * self.batch_size self.verbose_iter_ = self.verbose_iter_.tolist() if self.n_threads > 1: self.pool_ = ThreadPoolExecutor(self.n_threads) return self
from tempfile import TemporaryFile with TemporaryFile('w+') as f_obj: # same time a file read and write -> 'w+' f_obj.write("Life is cool.\n") f_obj.seek(4) # seeek to the begining data = f_obj.read() print(data)
def test_SequentialDesign_load_design(): "test the load_design method" ed = LatinHypercubeDesign(3) sd = SequentialDesign(ed, n_init=2, n_cand=3) with TemporaryFile() as tmp: np.savez(tmp, inputs=np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]), targets=np.array([2., 4.]), candidates=np.array([[0.7, 0.8, 0.9], [0.15, 0.25, 0.35], [0.45, 0.55, 0.65]])) tmp.seek(0) sd.load_design(tmp) assert_allclose(sd.inputs, np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]])) assert_allclose(sd.targets, np.array([2., 4.])) assert_allclose( sd.candidates, np.array([[0.7, 0.8, 0.9], [0.15, 0.25, 0.35], [0.45, 0.55, 0.65]])) assert sd.initialized assert sd.current_iteration == 2 # disagreement between base design and inputs ed = LatinHypercubeDesign(2) sd = SequentialDesign(ed, n_init=2, n_cand=3) with TemporaryFile() as tmp: np.savez(tmp, inputs=np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]), targets=np.array([2., 4.]), candidates=np.array([[0.7, 0.8, 0.9], [0.15, 0.25, 0.35], [0.45, 0.55, 0.65]])) tmp.seek(0) with pytest.raises(AssertionError): sd.load_design(tmp) # disagreement between shape of inputs and candidates ed = LatinHypercubeDesign(3) sd = SequentialDesign(ed, n_init=2, n_cand=3) with TemporaryFile() as tmp: np.savez(tmp, inputs=np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]), targets=np.array([2., 4.]), candidates=np.array([[0.7, 0.8], [0.15, 0.25], [0.45, 0.55]])) tmp.seek(0) with pytest.raises(AssertionError): sd.load_design(tmp) # error due to targets having a bad shape ed = LatinHypercubeDesign(3) sd = SequentialDesign(ed, n_init=2, n_cand=3) with TemporaryFile() as tmp: np.savez(tmp, inputs=np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]), targets=np.array([2., 4., 5.]), candidates=np.array([[0.7, 0.8], [0.15, 0.25], [0.45, 0.55]])) tmp.seek(0) with pytest.raises(AssertionError): sd.load_design(tmp) # error due to existing targets but no inputs ed = LatinHypercubeDesign(3) sd = SequentialDesign(ed, n_init=2, n_cand=3) with TemporaryFile() as tmp: np.savez(tmp, inputs=None, targets=np.array([2., 4., 5.]), candidates=None) tmp.seek(0) with pytest.raises(AssertionError): sd.load_design(tmp)
def execute_command_in_project(self, command, extra_environment_vars=None, timeout=None, **popen_kwargs): """ Execute a command in the context of the project :param command: the shell command to execute :type command: string :param extra_environment_vars: additional environment variables to set for command execution :type extra_environment_vars: dict[str, str] :param timeout: A maximum number of seconds before the process is terminated, or None for no timeout :type timeout: int | None :param popen_kwargs: additional keyword arguments to pass through to subprocess.Popen :type popen_kwargs: dict[str, mixed] :return: a tuple of (the string output from the command, the exit code of the command) :rtype: (string, int) """ environment_setter = self.shell_environment_command( extra_environment_vars) command = self.command_in_project('{} {}'.format( environment_setter, command)) self._logger.debug('Executing command in project: {}', command) # Redirect output to files instead of using pipes to avoid: https://github.com/box/ClusterRunner/issues/57 stdout_file = TemporaryFile() stderr_file = TemporaryFile() pipe = Popen( command, shell=True, stdout=stdout_file, stderr=stderr_file, start_new_session= True, # Starts a new process group (so we can kill it without killing clusterrunner). **popen_kwargs) clusterrunner_error_msgs = [] command_completed = False timeout_time = time.time() + (timeout or float('inf')) # Wait for the command to complete, but also periodically check the kill event flag to see if we should # terminate the process prematurely. while not command_completed and not self._kill_event.is_set( ) and time.time() < timeout_time: try: pipe.wait(timeout=1) command_completed = True # wait() didn't raise TimeoutExpired, so process has finished executing. except TimeoutExpired: continue except Exception as ex: # pylint: disable=broad-except error_message = 'Exception while waiting for process to finish.' self._logger.exception(error_message) clusterrunner_error_msgs.append( 'ClusterRunner: {} ({}: "{}")'.format( error_message, type(ex).__name__, ex)) break if not command_completed: # We've been signaled to terminate subprocesses, so terminate them. But we still collect stdout and stderr. # We must kill the entire process group since shell=True launches 'sh -c "cmd"' and just killing the pid # will kill only "sh" and not its child processes. # Note: We may lose buffered output from the subprocess that hasn't been flushed before termination. If we # want to prevent output buffering we should refactor this method to use pexpect. self._logger.warning('Terminating PID: {}, Command: "{}"', pipe.pid, command) try: # todo: os.killpg sends a SIGTERM to all processes in the process group. If the immediate child process # ("sh") dies but its child processes do not, we will leave them running orphaned. os.killpg(pipe.pid, signal.SIGTERM) except ( PermissionError, ProcessLookupError ) as ex: # os.killpg will raise if process has already ended self._logger.warning( 'Attempted to kill process group (pgid: {}) but raised {}: "{}".', pipe.pid, type(ex).__name__, ex) try: pipe.wait() except Exception as ex: # pylint: disable=broad-except error_message = 'Exception while waiting for terminated process to finish.' self._logger.exception(error_message) clusterrunner_error_msgs.append( 'ClusterRunner: {} ({}: "{}")'.format( error_message, type(ex).__name__, ex)) stdout, stderr = [ self._read_file_contents_and_close(f) for f in [stdout_file, stderr_file] ] exit_code = pipe.returncode if exit_code != 0: max_log_length = 300 logged_stdout, logged_stderr = stdout, stderr if len(stdout) > max_log_length: logged_stdout = '{}... (total stdout length: {})'.format( stdout[:max_log_length], len(stdout)) if len(stderr) > max_log_length: logged_stderr = '{}... (total stderr length: {})'.format( stderr[:max_log_length], len(stderr)) # Note we are intentionally not logging at error or warning level here. Interpreting a non-zero return code # as a failure is context-dependent, so we can't make that determination here. self._logger.notice( 'Command exited with non-zero exit code.\nCommand: {}\nExit code: {}\nStdout: {}\nStderr: {}\n', command, exit_code, logged_stdout, logged_stderr) else: self._logger.debug('Command completed with exit code {}.', exit_code) exit_code = exit_code if exit_code is not None else -1 # Make sure we always return an int. combined_command_output = '\n'.join([stdout, stderr] + clusterrunner_error_msgs) return combined_command_output, exit_code
def __init__(self, sess): self.sess = sess self.critic_learning_rate = 1e-4 #1e-4 #5e-5 #5e-5 self.actor_learning_rate = 1e-4 #1e-4 #5e-5 #5e-5 self.batch_size = 32 self.explore_rate = 0.002 #0.005 #0.003 #0.002 self.num_trial = 1 self.epsilon = .9 self.epsilon_decay = .999 #0.999 self.gamma = .90 self.tau = .01 self.obs_dim = 12 self.act_dim = 36 self.hidden_dim = 64 #data flow #last_hidden, obs, last_action -> hidden #hidden, action -> Q -> value #hidden -> policy -> action #next_obs, action, hidden -> next hidden tf.compat.v1.disable_eager_execution() self.ema = tf.train.ExponentialMovingAverage(decay=1 - self.tau) self.outfile = TemporaryFile() self.memory = deque(maxlen=5000) # ===================================================================== # # hidden Model # # ===================================================================== # self.obs_input, self.last_act_input, self.hidden_input, self.hidden_state, self.hidden_model = self.create_hidden_model( ) self.target_obs_input, self.target_last_act_input, self.target_hidden_input, self.target_hidden_state, self.target_hidden_model = self.create_hidden_model( ) # ===================================================================== # # Actor Model # # Chain rule: find the gradient of chaging the actor network params in # # getting closest to the final value network predictions, i.e. de/dA # # Calculate de/dA as = de/dC * dC/dA, where e is error, C critic, A act # # ===================================================================== # self.actor_model = self.create_actor_model() self.target_actor_model = self.create_target_actor_model() self.actor_critic_grad = tf.compat.v1.placeholder( tf.float32, [None, self.act_dim]) # where we will feed de/dC (from critic) actor_model_weights = self.actor_model.trainable_weights self.actor_grads = tf.gradients( self.actor_model.output, actor_model_weights, -self.actor_critic_grad) # dC/dA (from actor) grads = zip(self.actor_grads, actor_model_weights) self.optimize = tf.compat.v1.train.AdamOptimizer( self.actor_learning_rate).apply_gradients(grads) # ===================================================================== # # Critic Model # # ===================================================================== # self.critic_action_input, self.critic_model = self.create_critic_model( ) self.target_critic_act_input, self.target_critic_model = self.create_target_critic_model( ) self.critic_grads = tf.gradients( self.critic_model.output, self.critic_action_input ) # where we calcaulte de/dC for feeding above # Initialize for later gradient calculations self.sess.run(tf.compat.v1.global_variables_initializer())
def train_one_epoch(sess, ops, train_writer): """ ops: dict mapping from string to tf ops """ is_training = True outfile = TemporaryFile() # Shuffle train samples train_idxs = np.arange(0, len(TRAIN_DATASET)) print('length here:', len(TRAIN_DATASET)) np.random.shuffle(train_idxs) num_batches = len(TRAIN_DATASET) // BATCH_SIZE log_string('Len of dataset: %f' % len(TRAIN_DATASET)) log_string(str(datetime.now())) loss_sum = 0 for batch_idx in range(num_batches): start_idx = batch_idx * BATCH_SIZE end_idx = (batch_idx + 1) * BATCH_SIZE batch_data = get_cycle_batch(TRAIN_DATASET, train_idxs, start_idx, end_idx) feed_dict = { ops['pointclouds_pl']: batch_data, ops['is_training_pl']: is_training, } summary, step, _, grad_var_val, \ loss_val, pred_val, label_val, end_points_loss_val, \ end_points_f = sess.run([ops['merged'], ops['step'], ops['train_op'], ops['grad_var'][1:], ops['loss'], ops['pred'], ops['label'], ops['end_points_loss'], ops['end_points_f']], feed_dict=feed_dict) # print('Train end points loss val losses', end_points_loss_val) for g, v in grad_var_val: if np.isnan(g).any(): print('gradient is nan') ipdb.set_trace() if np.isnan(v).any(): print('variable is nan') ipdb.set_trace() if np.isnan(loss_val): print('>>>>>> NAN <<<<<<<<') ipdb.set_trace() # x = np.arange(16) # np.save('pointcloud', pred_val) # print ('point cloud value here:', ops['pointclouds_pl']) ### OPTIC FLOW HERE # print ('pred_val: ', pred_val.shape, type(pred_val)) train_writer.add_summary(summary, step) loss_sum += loss_val if (batch_idx + 1) % 1 == 0: log_string(' -- %03d / %03d --' % (batch_idx + 1, num_batches)) log_string('Cycle Train mean loss: %f' % (loss_sum / 2)) log_string('Cycle Train all losses {}'.format(end_points_loss_val)) loss_sum = 0
def test_write_file(self, props): with TemporaryFile(mode="w+") as f: props["a"] = "b" props.write(f) f.seek(0) assert f.read() == "a = b\n\n"
from xlwt import Workbook book = Workbook() sheet1 = book.add_sheet('result 1') for row_index in range(sheet0.nrows): keyword = sheet0.cell(row_index, 0).value params = {'query': keyword} enc_params = urllib.urlencode(params) request = urllib2.Request('http://search.chosun.com/search/' + 'total.search' + '?' + enc_params) request.add_header('User-agent', 'Mozilla/5.0') response = urllib2.urlopen(request) data = response.read().encode('utf-8') #aceept-encoding: gzip을 사용하지 않아서 관련된 코드와 모듈 모두 제거 if (data.find('<!-- 포커스 인물') != -1): sheet1.write(row_index, 4, 'yes') book.save('result.xls') book.save(TemporaryFile()) print 'yes' else: sheet1.write(row_index, 4, 'no') book.save('result.xls') book.save(TemporaryFile()) print 'no'
def download_lt(update=False): assert os.path.isdir(PACKAGE_PATH) old_path_list = [ path for path in glob.glob(os.path.join(PACKAGE_PATH, "LanguageTool*")) if os.path.isdir(path) ] if old_path_list and not update: return contents = "" for n, base_url in enumerate(BASE_URLS): try: with closing(urlopen(base_url)) as u: while True: data = u.read() if not data: break contents += data.decode() break except IOError as e: if n == len(BASE_URLS) - 1: raise href_format = r'<a href="(LanguageTool-(\d+.*?)\.{})">' matches = [(m.group(1), Version(m.group(2))) for m in re.finditer(href_format.format("zip"), contents, re.I)] if not matches: matches = [ (m.group(1), Version(m.group(2))) for m in re.finditer(href_format.format("oxt"), contents, re.I) ] filename, version = matches[-1] url = urljoin(base_url, filename) dirname = os.path.splitext(filename)[0] extract_path = os.path.join(PACKAGE_PATH, dirname) if extract_path in old_path_list: print("No update needed: {!r}".format(dirname)) return for old_path in old_path_list: match = re.search("LanguageTool-(\d+.*?)$", old_path) if match: current_version = Version(match.group(1)) try: version_test = current_version > version except TypeError: continue if version_test: print("Local version: {!r}, Remote version: {!r}".format( str(current_version), str(version))) return with closing(TemporaryFile()) as t: with closing(urlopen(url)) as u: content_len = int(u.headers["Content-Length"]) sys.stdout.write("Downloading {!r} ({:.1f} MiB)...\n".format( filename, content_len / 1048576.)) sys.stdout.flush() chunk_len = content_len // 100 data_len = 0 while True: data = u.read(chunk_len) if not data: break data_len += len(data) t.write(data) sys.stdout.write("\r{:.0%}".format( float(data_len) / content_len)) sys.stdout.flush() sys.stdout.write("\n") t.seek(0) for old_path in old_path_list: if os.path.isdir(old_path): shutil.rmtree(old_path) with closing(ZipFile(t)) as z: prefix = get_common_prefix(z) if prefix: z.extractall(PACKAGE_PATH) os.rename(os.path.join(PACKAGE_PATH, prefix), os.path.join(PACKAGE_PATH, dirname)) else: z.extractall(extract_path)
import numpy as np from tempfile import TemporaryFile outfile = TemporaryFile() x = np.arange(10) y = np.sin(x) np.savez(outfile, x, y) outfile.seek(0) # Only needed here to simulate closing & reopening file npzfile = np.load(outfile) npzfile.files npzfile['arr_0'] outfile = TemporaryFile() np.savez(outfile, x=x, y=y) outfile.seek(0) npzfile = np.load(outfile) npzfile.files npzfile['x']
def __init__(self, repo: mzbuild.Repository, name: str): self.name = name self.repo = repo self.images: List[mzbuild.Image] = [] self.workflows: Dict[str, Workflow] = {} default_tag = os.getenv(f"MZBUILD_TAG", None) if name in self.repo.compositions: self.path = self.repo.compositions[name] else: raise errors.UnknownComposition with open(self.path) as f: compose = yaml.safe_load(f) workflows = compose.pop("mzworkflows", None) if workflows is not None: # TODO: move this into the workflow so that it can use env vars that are # manually defined. workflows = _substitute_env_vars(workflows) for workflow_name, raw_w in workflows.items(): built_steps = [] for raw_step in raw_w["steps"]: step_name = raw_step.pop("step") step_ty = Steps.named(step_name) munged = { k.replace("-", "_"): v for k, v in raw_step.items() } try: step = step_ty(**munged) except TypeError as e: a = " ".join([f"{k}={v}" for k, v in munged.items()]) raise errors.BadSpec( f"Unable to construct {step_name} with args {a}: {e}" ) built_steps.append(step) env = raw_w.get("env") if not isinstance(env, dict) and env is not None: raise errors.BadSpec( f"Workflow {workflow_name} has wrong type for env: " f"expected mapping, got {type(env).__name__}: {env}", ) # ensure that integers (e.g. ports) are treated as env vars if isinstance(env, dict): env = {k: str(v) for k, v in env.items()} self.workflows[workflow_name] = Workflow( workflow_name, built_steps, env=env, composition=self, ) # Resolve all services that reference an `mzbuild` image to a specific # `image` reference. for config in compose["services"].values(): if "mzbuild" in config: image_name = config["mzbuild"] if image_name not in self.repo.images: raise errors.BadSpec( f"mzcompose: unknown image {image_name}") image = self.repo.images[image_name] override_tag = os.getenv(f"MZBUILD_{image.env_var_name()}_TAG", default_tag) if override_tag is not None: config["image"] = image.docker_name(override_tag) print( f"mzcompose: warning: overriding {image_name} image to tag {override_tag}", file=sys.stderr, ) del config["mzbuild"] else: self.images.append(image) if "propagate-uid-gid" in config: config["user"] = f"{os.getuid()}:{os.getgid()}" del config["propagate-uid-gid"] deps = self.repo.resolve_dependencies(self.images) for config in compose["services"].values(): if "mzbuild" in config: config["image"] = deps[config["mzbuild"]].spec() del config["mzbuild"] # Emit the munged configuration to a temporary file so that we can later # pass it to Docker Compose. tempfile = TemporaryFile() os.set_inheritable(tempfile.fileno(), True) yaml.dump(compose, tempfile, encoding="utf-8") # type: ignore tempfile.flush() self.file = tempfile
def __init__(self, mock_class, root_call, args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, restore_signals=True, start_new_session=False, pass_fds=(), encoding=None, errors=None, text=None): self.mock = Mock() self.class_instance_mock = mock_class.mock.Popen_instance #: A :func:`unittest.mock.call` representing the call made to instantiate #: this mock process. self.root_call = root_call #: The calls made on this mock process, represented using #: :func:`~unittest.mock.call` instances. self.calls = [] self.all_calls = mock_class.all_calls cmd = shell_join(args) behaviour = mock_class.commands.get(cmd, mock_class.default_behaviour) if behaviour is None: raise KeyError('Nothing specified for command %r' % cmd) if callable(behaviour): behaviour = behaviour(command=cmd, stdin=stdin) self.behaviour = behaviour stdout_value = behaviour.stdout stderr_value = behaviour.stderr if stderr == STDOUT: line_iterator = chain.from_iterable( zip_longest(stdout_value.splitlines(True), stderr_value.splitlines(True))) stdout_value = b''.join(l for l in line_iterator if l) stderr_value = None self.poll_count = behaviour.poll_count for name, option, mock_value in (('stdout', stdout, stdout_value), ('stderr', stderr, stderr_value)): value = None if option is PIPE: value = TemporaryFile() value.write(mock_value) value.flush() value.seek(0) if PY3 and (universal_newlines or text or encoding): value = TextIOWrapper(value, encoding=encoding, errors=errors) setattr(self, name, value) if stdin == PIPE: self.stdin = Mock() for method in 'write', 'close': record_writes = partial(self._record, ('stdin', method)) getattr(self.stdin, method).side_effect = record_writes self.pid = behaviour.pid #: The return code of this mock process. self.returncode = None if PY3: self.args = args
def test_writestr_permissions(self): for f in (TESTFN2, TemporaryFile(), StringIO()): self.zip_test_writestr_permissions(f, zipfile.ZIP_STORED)