def testLargeFileException(self):
for f in (TESTFN2, TemporaryFile(), StringIO()):
self.largeFileExceptionTest(f, zipfile.ZIP_STORED)
self.largeFileExceptionTest2(f, zipfile.ZIP_STORED)
def fftype(mol,
rtfFile=None,
prmFile=None,
method='GAFF2',
acCharges=None,
tmpDir=None,
netcharge=None):
"""
Assing atom types and force field parameters for a given molecule.
Additionally, atom masses and improper dihedral are set.
Optionally, atom charges can be set if `acCharges` is set (see below).
The assignment can be done:
1. For CHARMM CGenFF_2b6 with MATCH (method = 'CGenFF_2b6');
2. For AMBER GAFF with antechamber (method = 'GAFF');
3. For AMBER GAFF2 with antechamber (method = 'GAFF2');
Parameters
----------
mol : Molecule
Molecule to use for the assignment
rtfFile : str
Path to a RTF file from which to read the topology
prmFile : str
Path to a PRM file from which to read the parameters
method : str
Atomtyping assignment method.
Use :func:`fftype.listFftypemethods <htmd.parameterization.fftype.listFftypemethods>` to get a list of available
methods.
Default: :func:`fftype.defaultFftypemethod <htmd.parameterization.fftype.defaultFftypemethod>`
acCharges : str
Optionally assign charges with antechamber. Check `antechamber -L` for available options.
Note: only works for GAFF and GAFF2.
tmpDir: str
Directory for temporary files. If None, a directory is created and
deleted automatically.
netcharge : float
The net charge of the molecule.
Returns
-------
prm : :class:`ParameterSet <parmed.parameters.ParameterSet>` object
Returns a parmed ParameterSet object with the parameters.
mol : :class:`Molecule <moleculekit.molecule.Molecule>` object
The modified Molecule object with the matching atom types for the ParameterSet
"""
import parmed
if method not in fftypemethods:
raise ValueError('Invalid method {}. Available methods {}'.format(
method, ','.join(fftypemethods)))
if method == 'CGenFF_2b6' and acCharges:
raise ValueError('acCharges')
if netcharge is None:
netcharge = int(round(np.sum(mol.charge)))
logger.warning(
'Molecular charge is set to {} by adding up the atomic charges'.
format(netcharge))
if rtfFile and prmFile:
from htmd.parameterization.readers import readRTF
logger.info('Reading FF parameters from {} and {}'.format(
rtfFile, prmFile))
prm = parmed.charmm.CharmmParameterSet(rtfFile, prmFile)
names, elements, atomtypes, charges, masses, impropers = readRTF(
rtfFile)
else:
logger.info('Assigning atom types with {}'.format(method))
renamed_mol = _canonicalizeAtomNames(mol)
# Create a temporary directory
with TemporaryDirectory() as tmpdir:
# HACK to keep the files
tmpdir = tmpdir if tmpDir is None else tmpDir
logger.debug('Temporary directory: {}'.format(tmpdir))
if method in ('GAFF', 'GAFF2'):
from moleculekit.molecule import Molecule
from htmd.parameterization.readers import readPREPI, readFRCMOD
# Write the molecule to a file
renamed_mol.write(os.path.join(tmpdir, 'mol.mol2'))
atomtype = method.lower()
# Set arguments
cmd = [
'antechamber', '-at', atomtype, '-nc',
str(netcharge), '-fi', 'mol2', '-i', 'mol.mol2', '-fo',
'prepi', '-o', 'mol.prepi'
]
if acCharges is not None:
cmd += ['-c', acCharges]
# Run antechamber
with TemporaryFile() as stream:
if subprocess.call(
cmd, cwd=tmpdir, stdout=stream,
stderr=stream) != 0:
raise RuntimeError('"antechamber" failed')
stream.seek(0)
for line in stream.readlines():
logger.debug(line)
# Set arguments
cmd = [
'parmchk2', '-f', 'prepi', '-s', atomtype, '-i',
'mol.prepi', '-o', 'mol.frcmod', '-a', 'Y'
]
# Run parmchk2
with TemporaryFile() as stream:
if subprocess.call(
cmd, cwd=tmpdir, stdout=stream,
stderr=stream) != 0:
raise RuntimeError('"parmchk2" failed')
stream.seek(0)
for line in stream.readlines():
logger.debug(line)
# Check if antechamber did changes in atom names (and suggest the user to fix the names)
acmol = Molecule(os.path.join(tmpdir, 'NEWPDB.PDB'),
type='pdb')
acmol.name = np.array([n.upper()
for n in acmol.name]).astype(np.object)
changed_mol_acmol = np.setdiff1d(renamed_mol.name, acmol.name)
changed_acmol_mol = np.setdiff1d(acmol.name, renamed_mol.name)
if len(changed_mol_acmol) != 0 or len(changed_acmol_mol) != 0:
raise RuntimeError(
'Initial atom names {} were changed by antechamber to {}. '
'This probably means that the start of the atom name does not match '
'element symbol. '
'Please check the molecule.'
''.format(','.join(changed_mol_acmol),
','.join(changed_acmol_mol)))
# Read the results
prm = parmed.amber.AmberParameterSet(
os.path.join(tmpdir, 'mol.frcmod'))
names, atomtypes, charges, impropers = readPREPI(
renamed_mol, os.path.join(tmpdir, 'mol.prepi'))
masses, elements = readFRCMOD(
atomtypes, os.path.join(tmpdir, 'mol.frcmod'))
elif method == 'CGenFF_2b6':
from htmd.parameterization.readers import readRTF
# Write the molecule to a file
renamed_mol.write(os.path.join(tmpdir, 'mol.pdb'))
# Set arguments
cmd = [
'match-typer', '-charge',
str(netcharge), '-forcefield', 'top_all36_cgenff_new',
'mol.pdb'
]
# Run match-type
with TemporaryFile() as stream:
if subprocess.call(
cmd, cwd=tmpdir, stdout=stream,
stderr=stream) != 0:
raise RuntimeError('"match-typer" failed')
stream.seek(0)
for line in stream.readlines():
logger.debug(line)
prm = parmed.charmm.CharmmParameterSet(
os.path.join(tmpdir, 'mol.rtf'),
os.path.join(tmpdir, 'mol.prm'))
names, elements, atomtypes, charges, masses, impropers = readRTF(
os.path.join(tmpdir, 'mol.rtf'))
else:
raise ValueError('Invalid method {}'.format(method))
assert np.all(renamed_mol.name == names)
assert np.all(mol.element == elements)
mol = mol.copy()
mol.atomtype = atomtypes
mol.masses = masses
mol.impropers = impropers
if acCharges is not None:
mol.charge = charges
return prm, mol
past_weeks_4 = np.append(speed4.T, speed4.T, axis=1)
past_weeks = np.append(past_weeks_1, past_weeks_2, axis=0)
past_weeks = np.append(past_weeks, past_weeks_3, axis=0)
past_weeks = np.append(past_weeks, past_weeks_4, axis=0)
patterns = past_weeks
f_particles = np.random.choice([0, 1], size=(8, 100))
fantasy_particles = np.append(f_particles, f_particles, axis=1)
iterations = 100
#################################################################
#Learn the distribution
learn(patterns, fantasy_particles, modelParameters, units, numConnections,
connections_index, iterations)
elapsed_time = time.time() - start_time
################################################################
#Test the model's prediction
start_time = time.time()
recovered = recall(speed_test[:, 1].T, units, modelParameters)
elapsed_time_testing = time.time() - start_time
print("Recovered: ", recovered[42:45])
print("Actual Speed: ", speed_test[42:45, 1])
print elapsed_time, elapsed_time_testing
from tempfile import TemporaryFile
recoveredSpeed = TemporaryFile()
np.savez('recoveredSpeed_fA_2',
recovered=recovered,
ActualSpeed=speed_test[:, 1])
def run(self, data, store, signal, context, **kwargs):
""" The main run method of the Python task.
Args:
data (MultiTaskData): The data object that has been passed from the
predecessor task.
store (DataStoreDocument): The persistent data store object that allows the
task to store data for access across the current
workflow run.
signal (TaskSignal): The signal object for tasks. It wraps the construction
and sending of signals into easy to use methods.
context (TaskContext): The context in which the tasks runs.
Returns:
Action: An Action object containing the data that should be passed on
to the next task and optionally a list of successor tasks that
should be executed.
"""
params = self.params.eval(data, store, exclude=['command'])
capture_stdout = self._callback_stdout is not None or params.capture_stdout
capture_stderr = self._callback_stderr is not None or params.capture_stderr
stdout_file = TemporaryFile() if params.capture_stdout else None
stderr_file = TemporaryFile() if params.capture_stderr else None
stdout = PIPE if capture_stdout else None
stderr = PIPE if capture_stderr else None
# change the user or group under which the process should run
if params.user is not None or params.group is not None:
pre_exec = self._run_as(params.user, params.group)
else:
pre_exec = None
# call the command
proc = Popen(self.params.eval_single('command', data, store),
cwd=params.cwd,
shell=True,
env=params.env,
preexec_fn=pre_exec,
stdout=stdout,
stderr=stderr,
stdin=PIPE if params.stdin is not None else None)
# if input is available, send it to the process
if params.stdin is not None:
proc.stdin.write(params.stdin.encode(sys.getfilesystemencoding()))
# send a notification that the process has been started
try:
if self._callback_process is not None:
self._callback_process(proc.pid, data, store, signal, context)
except (StopTask, AbortWorkflow):
proc.terminate()
raise
# send the output handling to a thread
if capture_stdout or capture_stderr:
output_reader = BashTaskOutputReader(proc, stdout_file,
stderr_file,
self._callback_stdout,
self._callback_stderr,
params.refresh_time, data,
store, signal, context)
output_reader.start()
else:
output_reader = None
# wait for the process to complete and watch for a stop signal
while proc.poll() is None or\
(output_reader is not None and output_reader.is_alive()):
sleep(params.refresh_time)
if signal.is_stopped:
proc.terminate()
if output_reader is not None:
output_reader.join()
data = output_reader.data
# if a stop or abort exception was raised, stop the bash process and re-raise
if output_reader.exc_obj is not None:
if proc.poll() is None:
proc.terminate()
raise output_reader.exc_obj
# send a notification that the process has completed
if self._callback_end is not None:
if stdout_file is not None:
stdout_file.seek(0)
if stderr_file is not None:
stderr_file.seek(0)
self._callback_end(proc.returncode, stdout_file, stderr_file, data,
store, signal, context)
if stdout_file is not None:
stdout_file.close()
if stderr_file is not None:
stderr_file.close()
return Action(data)
#!/usr/bin/env python3
# encoding=utf-8
from tempfile import TemporaryFile, NamedTemporaryFile, TemporaryDirectory
with TemporaryFile('w+t') as f:
f.write('heyheyhey')
f.seek(0)
data = f.read()
print(data)
with NamedTemporaryFile('w+t') as f:
f.write('I\'m NamedTemporaryFile')
f.seek(0)
data = f.read()
print(data)
print(f.name)
with TemporaryDirectory() as d:
print('dirname: ' + d)
#!/usr/bin/env python
# -*- coding:utf-8 -*-
# __author__ = 'liao gao xiang'
import tempfile
from tempfile import TemporaryFile, TemporaryDirectory, NamedTemporaryFile
# 你需要在程序执行时创建一个临时文件或目录，并希望使用完之后可以自动销毁掉。
with TemporaryFile('w+t', encoding='utf-8') as f:
f.write('liaogx\n')
f.write('Python')
f.seek(0)
print(f.read())
"""
TemporaryFile() 的第一个参数是文件模式，通常来讲文本模式使用 w+t ，二进
制模式使用 w+b 。这个模式同时支持读和写操作，在这里是很有用的，因为当你关闭
文件去改变模式的时候，文件实际上已经不存在了
"""
# 命名的临时文件，且关闭文件时不删除
with NamedTemporaryFile('w+t', encoding='utf-8', delete=False) as f:
f.write(f.name)
# 创建临时目录
with TemporaryDirectory() as dirname:
print('dirname is: ', dirname)
# 通常来讲，临时文件在系统默认的位置被创建，比如 /var/tmp 或类似的地方。为
# 了获取真实的位置，可以使用 tempfile.gettempdir() 函数
def check_example(package, filename):
example_dir = path.join(examples, package)
expected_dir = path.join(expected, package)
expected_base = path.join(expected_dir, path.splitext(filename)[0])
try:
with TempDirectory() as actual:
# copy files to the directory
copy(path.join(example_dir, filename), actual.path)
for pattern in ('*.xls', '*.bmp'):
for fixture in glob(path.join(example_dir, pattern)):
copy(fixture, actual.path)
os.chdir(actual.path)
output = TemporaryFile('w+')
# run the example
before_listing = set(os.listdir(actual.path))
call([runner, filename], stdout=output, stderr=STDOUT)
after_listing = set(os.listdir(actual.path))
# check the console output
output.seek(0)
actual_output = output.read().strip().replace('\r', '')
for re, rp in sub_res:
actual_output = re.sub(rp, actual_output)
expected_path = expected_base + '.txt'
if not path.exists(expected_path):
expected_output = ''
else:
expected_output = open(expected_path).read().strip().replace(
'\r', '')
compare(expected_output, actual_output)
# check the files created
created = after_listing.difference(before_listing)
expected_names = set()
if os.path.exists(expected_base):
expected_names = set(os.listdir(expected_base))
for name in created:
with open(path.join(actual.path, name), 'rb') as af:
actual_data = af.read()
if name in expected_names:
expected_path = path.join(expected_base, name)
expected_data = open(expected_path, 'rb').read()
expected_names.remove(name)
if actual_data != expected_data:
if environ.get('REPLACE_EXAMPLES'):
with open(expected_path, 'wb') as new_expected:
new_expected.write(actual_data)
compare(
get_biff_records(expected_data),
get_biff_records(actual_data),
)
else:
raise AssertionError("unexpected output: %s" % name)
for name in expected_names:
if name != '.svn':
print created
raise AssertionError("expected output missing: %s" % name)
finally:
os.chdir(initial)
def persist_lines_job(project_id, dataset_id, lines=None):
state = None
schemas = {}
key_properties = {}
tables = {}
rows = {}
errors = {}
bigquery_client = bigquery.Client(project=project_id)
# try:
# dataset = bigquery_client.create_dataset(Dataset(dataset_ref)) or Dataset(dataset_ref)
# except exceptions.Conflict:
# pass
for line in lines:
try:
msg = singer.parse_message(line)
except json.decoder.JSONDecodeError:
logger.error("Unable to parse:\n{}".format(line))
raise
if isinstance(msg, singer.RecordMessage):
if msg.stream not in schemas:
raise Exception("A record for stream {} was encountered before a corresponding schema".format(msg.stream))
schema = schemas[msg.stream]
validate(msg.record, schema)
dat = bytes(str(json.loads(json.dumps(msg.record), object_pairs_hook=clear_dict_hook)) + '\n', 'UTF-8')
rows[msg.stream].write(dat)
#rows[msg.stream].write(bytes(str(msg.record) + '\n', 'UTF-8'))
state = None
elif isinstance(msg, singer.StateMessage):
logger.debug('Setting state to {}'.format(msg.value))
state = msg.value
elif isinstance(msg, singer.SchemaMessage):
table = msg.stream
schemas[table] = msg.schema
key_properties[table] = msg.key_properties
#tables[table] = bigquery.Table(dataset.table(table), schema=build_schema(schemas[table]))
rows[table] = TemporaryFile(mode='w+b')
errors[table] = None
# try:
# tables[table] = bigquery_client.create_table(tables[table])
# except exceptions.Conflict:
# pass
else:
raise Exception("Unrecognized message {}".format(msg))
for table in rows.keys():
table_ref = bigquery_client.dataset(dataset_id).table(table)
SCHEMA = build_schema(schemas[table])
load_config = LoadJobConfig()
load_config.schema = SCHEMA
load_config.source_format = SourceFormat.NEWLINE_DELIMITED_JSON
rows[table].seek(0)
logger.info("loading {} to Bigquery.\n".format(table))
load_job = bigquery_client.load_table_from_file(
rows[table], table_ref, job_config=load_config)
logger.info("loading job {}".format(load_job.job_id))
logger.info(load_job.result())
# for table in errors.keys():
# if not errors[table]:
# print('Loaded {} row(s) into {}:{}'.format(rows[table], dataset_id, table), tables[table].path)
# else:
# print('Errors:', errors[table], sep=" ")
return state
def get_torch_object_bytes(obj):
with TemporaryFile() as f:
torch_save(obj, f)
f.seek(0)
b = f.read()
return b
def telegramImgSend(url):
with TemporaryFile() as f:
f.write(requests.get(url).content)
f.seek(0)
bot.sendPhoto(TELEGRAM_CHAT_ID, f)
def start_services(self):
self.server = subprocess.Popen(['python', abspath(__file__)],
stdout=TemporaryFile(),
stderr=subprocess.STDOUT)
def testRandomOpenStored(self):
for f in (TESTFN2, TemporaryFile(), StringIO()):
self.zipRandomOpenTest(f, zipfile.ZIP_STORED)
def testWriteNonPyfile(self):
zipfp = zipfile.PyZipFile(TemporaryFile(), "w")
file(TESTFN, 'w').write('most definitely not a python file')
self.assertRaises(RuntimeError, zipfp.writepy, TESTFN)
os.remove(TESTFN)
def testDeflated(self):
for f in (TESTFN2, TemporaryFile(), StringIO()):
self.zipTest(f, zipfile.ZIP_DEFLATED)
from gtts import gTTS
import os
from pygame import mixer
from tempfile import TemporaryFile
from io import BytesIO
import time
tts = gTTS(text='Hello world.', lang='en')
#tts.save("hello.mp3")
#os.system("mpg321 hello.mp3")
print("init mixer")
mixer.init()
#mp3_fp = BytesIO()
mp3_fp = TemporaryFile()
print("Write to bytes like")
tts.write_to_fp(mp3_fp)
mp3_fp.seek(0)
print("load file in mixer")
mixer.music.load(mp3_fp)
print("play sound")
mixer.music.play()
while mixer.music.get_busy():
time.sleep(0.1)
print("Done")
def load_torch_object_bytes(b):
with TemporaryFile() as f:
f.write(b)
f.seek(0)
obj = torch_load(f)
return obj
def draw(self, context, t):
mode = self.get_string("mode", t, "clamp")
img_ease = self.get_bool("img_ease", t, True)
img_speed = self.get_number("img_speed", t, 1)
if img_ease:
img = self.get_image("image_surfaces", self.interpolate(self.no_interp_time * img_speed), None, mode)
else:
img = self.get_image("image_surfaces", self.no_interp_time * img_speed, None, mode)
x = self.get_number("x", t, 100)
y = self.get_number("y", t, 100)
w = self.get_number("w", t, None)
h = self.get_number("h", t, None)
scale_x = self.get_number("scale_x", t, 1)
scale_y = self.get_number("scale_y", t, 1)
alpha = self.get_number("alpha", t, 1)
rotation = rad(self.get_number("rotation", t, 0))
tint = self.get_color("tint", t, None)
tint_op = self.get_cairo_constant("operator", "tint_op", t, cairo.OPERATOR_HSL_COLOR)
if not img:
return
if w is None:
w = img.get_width()
if h is None:
h = img.get_height()
if tint is not None:
# necessary to prevent reuse
f = TemporaryFile()
img.write_to_png(f)
f.seek(0)
_img = cairo.ImageSurface.create_from_png(f)
b = cairo.Context(_img)
# create alpha mask
mask = cairo.ImageSurface(cairo.FORMAT_A8, img.get_width(), img.get_height())
maskctx = cairo.Context(mask)
maskctx.set_source_surface(img)
maskctx.paint()
# paint
b.set_operator(tint_op)
b.set_source_rgba(*tint)
b.mask(cairo.SurfacePattern(mask))
else:
_img = img
context.translate(x, y)
context.scale(scale_x, scale_y)
context.rotate(rotation)
if self.get_bool("centered", t, True):
context.translate(-0.5 * w, -0.5 * h)
context.scale(w / _img.get_width(), h / _img.get_height())
context.set_source_surface(_img)
context.paint_with_alpha(alpha)
def prepare(self, n_samples=None, n_features=None, dtype=None, X=None):
"""
Init estimator attributes based on input shape and type.
Parameters
----------
n_samples: int,
n_features: int,
dtype: dtype in np.float32, np.float64
to use in the estimator. Override X.dtype if provided
X: ndarray, shape (> n_components, n_features)
Array to use to determine shape and types, and init dictionary if
provided
Returns
-------
self
"""
if X is not None:
X = check_array(X, order='C', dtype=[np.float32, np.float64])
if dtype is None:
dtype = X.dtype
# Transpose to fit usual column streaming
this_n_samples = X.shape[0]
if n_samples is None:
n_samples = this_n_samples
if n_features is None:
n_features = X.shape[1]
else:
if n_features != X.shape[1]:
raise ValueError('n_features and X does not match')
else:
if n_features is None or n_samples is None:
raise ValueError('Either provide'
'shape or data to function prepare.')
if dtype is None:
dtype = np.float64
elif dtype not in [np.float32, np.float64]:
return ValueError('dtype should be float32 or float64')
# Regression statistics
if self.G_agg == 'average':
with TemporaryFile() as self.G_average_mmap_:
self.G_average_mmap_ = TemporaryFile()
self.G_average_ = np.memmap(self.G_average_mmap_,
mode='w+',
shape=(n_samples,
self.n_components,
self.n_components),
dtype=dtype)
atexit.register(self._exit)
self.Dx_average_ = np.zeros((n_samples, self.n_components),
dtype=dtype)
# Dictionary statistics
self.C_ = np.zeros((self.n_components, self.n_components), dtype=dtype)
self.B_ = np.zeros((self.n_components, n_features), dtype=dtype)
self.gradient_ = np.zeros((self.n_components, n_features),
dtype=dtype,
order='F')
self.random_state = check_random_state(self.random_state)
if X is None:
self.components_ = np.empty((self.n_components, n_features),
dtype=dtype)
self.components_[:, :] = self.random_state.randn(
self.n_components, n_features)
else:
random_idx = self.random_state.permutation(
this_n_samples)[:self.n_components]
self.components_ = check_array(X[random_idx],
dtype=dtype.type,
copy=True)
if self.comp_pos:
self.components_[self.components_ <= 0] = \
- self.components_[self.components_ <= 0]
for i in range(self.n_components):
enet_scale(self.components_[i],
l1_ratio=self.comp_l1_ratio,
radius=1)
self.code_ = np.ones((n_samples, self.n_components), dtype=dtype)
self.labels_ = np.arange(n_samples)
self.comp_norm_ = np.zeros(self.n_components, dtype=dtype)
if self.G_agg == 'full':
self.G_ = self.components_.dot(self.components_.T)
self.n_iter_ = 0
self.sample_n_iter_ = np.zeros(n_samples, dtype='int')
self.random_state = check_random_state(self.random_state)
random_seed = self.random_state.randint(MAX_INT)
self.feature_sampler_ = Sampler(n_features, self.rand_size,
self.replacement, random_seed)
if self.verbose:
log_lim = log(n_samples * self.n_epochs / self.batch_size, 10)
self.verbose_iter_ = (np.logspace(
0, log_lim, self.verbose, base=10) - 1) * self.batch_size
self.verbose_iter_ = self.verbose_iter_.tolist()
if self.n_threads > 1:
self.pool_ = ThreadPoolExecutor(self.n_threads)
return self
from tempfile import TemporaryFile
with TemporaryFile('w+') as f_obj: # same time a file read and write -> 'w+'
f_obj.write("Life is cool.\n")
f_obj.seek(4) # seeek to the begining
data = f_obj.read()
print(data)
def test_SequentialDesign_load_design():
"test the load_design method"
ed = LatinHypercubeDesign(3)
sd = SequentialDesign(ed, n_init=2, n_cand=3)
with TemporaryFile() as tmp:
np.savez(tmp,
inputs=np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]),
targets=np.array([2., 4.]),
candidates=np.array([[0.7, 0.8, 0.9], [0.15, 0.25, 0.35],
[0.45, 0.55, 0.65]]))
tmp.seek(0)
sd.load_design(tmp)
assert_allclose(sd.inputs, np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]))
assert_allclose(sd.targets, np.array([2., 4.]))
assert_allclose(
sd.candidates,
np.array([[0.7, 0.8, 0.9], [0.15, 0.25, 0.35], [0.45, 0.55, 0.65]]))
assert sd.initialized
assert sd.current_iteration == 2
# disagreement between base design and inputs
ed = LatinHypercubeDesign(2)
sd = SequentialDesign(ed, n_init=2, n_cand=3)
with TemporaryFile() as tmp:
np.savez(tmp,
inputs=np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]),
targets=np.array([2., 4.]),
candidates=np.array([[0.7, 0.8, 0.9], [0.15, 0.25, 0.35],
[0.45, 0.55, 0.65]]))
tmp.seek(0)
with pytest.raises(AssertionError):
sd.load_design(tmp)
# disagreement between shape of inputs and candidates
ed = LatinHypercubeDesign(3)
sd = SequentialDesign(ed, n_init=2, n_cand=3)
with TemporaryFile() as tmp:
np.savez(tmp,
inputs=np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]),
targets=np.array([2., 4.]),
candidates=np.array([[0.7, 0.8], [0.15, 0.25], [0.45, 0.55]]))
tmp.seek(0)
with pytest.raises(AssertionError):
sd.load_design(tmp)
# error due to targets having a bad shape
ed = LatinHypercubeDesign(3)
sd = SequentialDesign(ed, n_init=2, n_cand=3)
with TemporaryFile() as tmp:
np.savez(tmp,
inputs=np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]),
targets=np.array([2., 4., 5.]),
candidates=np.array([[0.7, 0.8], [0.15, 0.25], [0.45, 0.55]]))
tmp.seek(0)
with pytest.raises(AssertionError):
sd.load_design(tmp)
# error due to existing targets but no inputs
ed = LatinHypercubeDesign(3)
sd = SequentialDesign(ed, n_init=2, n_cand=3)
with TemporaryFile() as tmp:
np.savez(tmp,
inputs=None,
targets=np.array([2., 4., 5.]),
candidates=None)
tmp.seek(0)
with pytest.raises(AssertionError):
sd.load_design(tmp)
def execute_command_in_project(self,
command,
extra_environment_vars=None,
timeout=None,
**popen_kwargs):
"""
Execute a command in the context of the project
:param command: the shell command to execute
:type command: string
:param extra_environment_vars: additional environment variables to set for command execution
:type extra_environment_vars: dict[str, str]
:param timeout: A maximum number of seconds before the process is terminated, or None for no timeout
:type timeout: int | None
:param popen_kwargs: additional keyword arguments to pass through to subprocess.Popen
:type popen_kwargs: dict[str, mixed]
:return: a tuple of (the string output from the command, the exit code of the command)
:rtype: (string, int)
"""
environment_setter = self.shell_environment_command(
extra_environment_vars)
command = self.command_in_project('{} {}'.format(
environment_setter, command))
self._logger.debug('Executing command in project: {}', command)
# Redirect output to files instead of using pipes to avoid: https://github.com/box/ClusterRunner/issues/57
stdout_file = TemporaryFile()
stderr_file = TemporaryFile()
pipe = Popen(
command,
shell=True,
stdout=stdout_file,
stderr=stderr_file,
start_new_session=
True, # Starts a new process group (so we can kill it without killing clusterrunner).
**popen_kwargs)
clusterrunner_error_msgs = []
command_completed = False
timeout_time = time.time() + (timeout or float('inf'))
# Wait for the command to complete, but also periodically check the kill event flag to see if we should
# terminate the process prematurely.
while not command_completed and not self._kill_event.is_set(
) and time.time() < timeout_time:
try:
pipe.wait(timeout=1)
command_completed = True # wait() didn't raise TimeoutExpired, so process has finished executing.
except TimeoutExpired:
continue
except Exception as ex: # pylint: disable=broad-except
error_message = 'Exception while waiting for process to finish.'
self._logger.exception(error_message)
clusterrunner_error_msgs.append(
'ClusterRunner: {} ({}: "{}")'.format(
error_message,
type(ex).__name__, ex))
break
if not command_completed:
# We've been signaled to terminate subprocesses, so terminate them. But we still collect stdout and stderr.
# We must kill the entire process group since shell=True launches 'sh -c "cmd"' and just killing the pid
# will kill only "sh" and not its child processes.
# Note: We may lose buffered output from the subprocess that hasn't been flushed before termination. If we
# want to prevent output buffering we should refactor this method to use pexpect.
self._logger.warning('Terminating PID: {}, Command: "{}"',
pipe.pid, command)
try:
# todo: os.killpg sends a SIGTERM to all processes in the process group. If the immediate child process
# ("sh") dies but its child processes do not, we will leave them running orphaned.
os.killpg(pipe.pid, signal.SIGTERM)
except (
PermissionError, ProcessLookupError
) as ex: # os.killpg will raise if process has already ended
self._logger.warning(
'Attempted to kill process group (pgid: {}) but raised {}: "{}".',
pipe.pid,
type(ex).__name__, ex)
try:
pipe.wait()
except Exception as ex: # pylint: disable=broad-except
error_message = 'Exception while waiting for terminated process to finish.'
self._logger.exception(error_message)
clusterrunner_error_msgs.append(
'ClusterRunner: {} ({}: "{}")'.format(
error_message,
type(ex).__name__, ex))
stdout, stderr = [
self._read_file_contents_and_close(f)
for f in [stdout_file, stderr_file]
]
exit_code = pipe.returncode
if exit_code != 0:
max_log_length = 300
logged_stdout, logged_stderr = stdout, stderr
if len(stdout) > max_log_length:
logged_stdout = '{}... (total stdout length: {})'.format(
stdout[:max_log_length], len(stdout))
if len(stderr) > max_log_length:
logged_stderr = '{}... (total stderr length: {})'.format(
stderr[:max_log_length], len(stderr))
# Note we are intentionally not logging at error or warning level here. Interpreting a non-zero return code
# as a failure is context-dependent, so we can't make that determination here.
self._logger.notice(
'Command exited with non-zero exit code.\nCommand: {}\nExit code: {}\nStdout: {}\nStderr: {}\n',
command, exit_code, logged_stdout, logged_stderr)
else:
self._logger.debug('Command completed with exit code {}.',
exit_code)
exit_code = exit_code if exit_code is not None else -1 # Make sure we always return an int.
combined_command_output = '\n'.join([stdout, stderr] +
clusterrunner_error_msgs)
return combined_command_output, exit_code
def __init__(self, sess):
self.sess = sess
self.critic_learning_rate = 1e-4 #1e-4 #5e-5 #5e-5
self.actor_learning_rate = 1e-4 #1e-4 #5e-5 #5e-5
self.batch_size = 32
self.explore_rate = 0.002 #0.005 #0.003 #0.002
self.num_trial = 1
self.epsilon = .9
self.epsilon_decay = .999 #0.999
self.gamma = .90
self.tau = .01
self.obs_dim = 12
self.act_dim = 36
self.hidden_dim = 64
#data flow
#last_hidden, obs, last_action -> hidden
#hidden, action -> Q -> value
#hidden -> policy -> action
#next_obs, action, hidden -> next hidden
tf.compat.v1.disable_eager_execution()
self.ema = tf.train.ExponentialMovingAverage(decay=1 - self.tau)
self.outfile = TemporaryFile()
self.memory = deque(maxlen=5000)
# ===================================================================== #
# hidden Model #
# ===================================================================== #
self.obs_input, self.last_act_input, self.hidden_input, self.hidden_state, self.hidden_model = self.create_hidden_model(
)
self.target_obs_input, self.target_last_act_input, self.target_hidden_input, self.target_hidden_state, self.target_hidden_model = self.create_hidden_model(
)
# ===================================================================== #
# Actor Model #
# Chain rule: find the gradient of chaging the actor network params in #
# getting closest to the final value network predictions, i.e. de/dA #
# Calculate de/dA as = de/dC * dC/dA, where e is error, C critic, A act #
# ===================================================================== #
self.actor_model = self.create_actor_model()
self.target_actor_model = self.create_target_actor_model()
self.actor_critic_grad = tf.compat.v1.placeholder(
tf.float32,
[None, self.act_dim]) # where we will feed de/dC (from critic)
actor_model_weights = self.actor_model.trainable_weights
self.actor_grads = tf.gradients(
self.actor_model.output, actor_model_weights,
-self.actor_critic_grad) # dC/dA (from actor)
grads = zip(self.actor_grads, actor_model_weights)
self.optimize = tf.compat.v1.train.AdamOptimizer(
self.actor_learning_rate).apply_gradients(grads)
# ===================================================================== #
# Critic Model #
# ===================================================================== #
self.critic_action_input, self.critic_model = self.create_critic_model(
)
self.target_critic_act_input, self.target_critic_model = self.create_target_critic_model(
)
self.critic_grads = tf.gradients(
self.critic_model.output, self.critic_action_input
) # where we calcaulte de/dC for feeding above
# Initialize for later gradient calculations
self.sess.run(tf.compat.v1.global_variables_initializer())
def train_one_epoch(sess, ops, train_writer):
""" ops: dict mapping from string to tf ops """
is_training = True
outfile = TemporaryFile()
# Shuffle train samples
train_idxs = np.arange(0, len(TRAIN_DATASET))
print('length here:', len(TRAIN_DATASET))
np.random.shuffle(train_idxs)
num_batches = len(TRAIN_DATASET) // BATCH_SIZE
log_string('Len of dataset: %f' % len(TRAIN_DATASET))
log_string(str(datetime.now()))
loss_sum = 0
for batch_idx in range(num_batches):
start_idx = batch_idx * BATCH_SIZE
end_idx = (batch_idx + 1) * BATCH_SIZE
batch_data = get_cycle_batch(TRAIN_DATASET, train_idxs, start_idx,
end_idx)
feed_dict = {
ops['pointclouds_pl']: batch_data,
ops['is_training_pl']: is_training,
}
summary, step, _, grad_var_val, \
loss_val, pred_val, label_val, end_points_loss_val, \
end_points_f = sess.run([ops['merged'], ops['step'],
ops['train_op'],
ops['grad_var'][1:],
ops['loss'],
ops['pred'],
ops['label'],
ops['end_points_loss'],
ops['end_points_f']], feed_dict=feed_dict)
# print('Train end points loss val losses', end_points_loss_val)
for g, v in grad_var_val:
if np.isnan(g).any():
print('gradient is nan')
ipdb.set_trace()
if np.isnan(v).any():
print('variable is nan')
ipdb.set_trace()
if np.isnan(loss_val):
print('>>>>>> NAN <<<<<<<<')
ipdb.set_trace()
# x = np.arange(16)
# np.save('pointcloud', pred_val)
# print ('point cloud value here:', ops['pointclouds_pl'])
### OPTIC FLOW HERE
# print ('pred_val: ', pred_val.shape, type(pred_val))
train_writer.add_summary(summary, step)
loss_sum += loss_val
if (batch_idx + 1) % 1 == 0:
log_string(' -- %03d / %03d --' % (batch_idx + 1, num_batches))
log_string('Cycle Train mean loss: %f' % (loss_sum / 2))
log_string('Cycle Train all losses {}'.format(end_points_loss_val))
loss_sum = 0
def test_write_file(self, props):
with TemporaryFile(mode="w+") as f:
props["a"] = "b"
props.write(f)
f.seek(0)
assert f.read() == "a = b\n\n"
from xlwt import Workbook
book = Workbook()
sheet1 = book.add_sheet('result 1')
for row_index in range(sheet0.nrows):
keyword = sheet0.cell(row_index, 0).value
params = {'query': keyword}
enc_params = urllib.urlencode(params)
request = urllib2.Request('http://search.chosun.com/search/' +
'total.search' + '?' + enc_params)
request.add_header('User-agent', 'Mozilla/5.0')
response = urllib2.urlopen(request)
data = response.read().encode('utf-8')
#aceept-encoding: gzip을 사용하지 않아서 관련된 코드와 모듈 모두 제거
if (data.find('<!-- 포커스 인물') != -1):
sheet1.write(row_index, 4, 'yes')
book.save('result.xls')
book.save(TemporaryFile())
print 'yes'
else:
sheet1.write(row_index, 4, 'no')
book.save('result.xls')
book.save(TemporaryFile())
print 'no'
def download_lt(update=False):
assert os.path.isdir(PACKAGE_PATH)
old_path_list = [
path for path in glob.glob(os.path.join(PACKAGE_PATH, "LanguageTool*"))
if os.path.isdir(path)
]
if old_path_list and not update:
return
contents = ""
for n, base_url in enumerate(BASE_URLS):
try:
with closing(urlopen(base_url)) as u:
while True:
data = u.read()
if not data:
break
contents += data.decode()
break
except IOError as e:
if n == len(BASE_URLS) - 1:
raise
href_format = r'<a href="(LanguageTool-(\d+.*?)\.{})">'
matches = [(m.group(1), Version(m.group(2)))
for m in re.finditer(href_format.format("zip"), contents, re.I)]
if not matches:
matches = [
(m.group(1), Version(m.group(2)))
for m in re.finditer(href_format.format("oxt"), contents, re.I)
]
filename, version = matches[-1]
url = urljoin(base_url, filename)
dirname = os.path.splitext(filename)[0]
extract_path = os.path.join(PACKAGE_PATH, dirname)
if extract_path in old_path_list:
print("No update needed: {!r}".format(dirname))
return
for old_path in old_path_list:
match = re.search("LanguageTool-(\d+.*?)$", old_path)
if match:
current_version = Version(match.group(1))
try:
version_test = current_version > version
except TypeError:
continue
if version_test:
print("Local version: {!r}, Remote version: {!r}".format(
str(current_version), str(version)))
return
with closing(TemporaryFile()) as t:
with closing(urlopen(url)) as u:
content_len = int(u.headers["Content-Length"])
sys.stdout.write("Downloading {!r} ({:.1f} MiB)...\n".format(
filename, content_len / 1048576.))
sys.stdout.flush()
chunk_len = content_len // 100
data_len = 0
while True:
data = u.read(chunk_len)
if not data:
break
data_len += len(data)
t.write(data)
sys.stdout.write("\r{:.0%}".format(
float(data_len) / content_len))
sys.stdout.flush()
sys.stdout.write("\n")
t.seek(0)
for old_path in old_path_list:
if os.path.isdir(old_path):
shutil.rmtree(old_path)
with closing(ZipFile(t)) as z:
prefix = get_common_prefix(z)
if prefix:
z.extractall(PACKAGE_PATH)
os.rename(os.path.join(PACKAGE_PATH, prefix),
os.path.join(PACKAGE_PATH, dirname))
else:
z.extractall(extract_path)
import numpy as np from tempfile import TemporaryFile outfile = TemporaryFile() x = np.arange(10) y = np.sin(x) np.savez(outfile, x, y) outfile.seek(0) # Only needed here to simulate closing & reopening file npzfile = np.load(outfile) npzfile.files npzfile['arr_0'] outfile = TemporaryFile() np.savez(outfile, x=x, y=y) outfile.seek(0) npzfile = np.load(outfile) npzfile.files npzfile['x']
def __init__(self, repo: mzbuild.Repository, name: str):
self.name = name
self.repo = repo
self.images: List[mzbuild.Image] = []
self.workflows: Dict[str, Workflow] = {}
default_tag = os.getenv(f"MZBUILD_TAG", None)
if name in self.repo.compositions:
self.path = self.repo.compositions[name]
else:
raise errors.UnknownComposition
with open(self.path) as f:
compose = yaml.safe_load(f)
workflows = compose.pop("mzworkflows", None)
if workflows is not None:
# TODO: move this into the workflow so that it can use env vars that are
# manually defined.
workflows = _substitute_env_vars(workflows)
for workflow_name, raw_w in workflows.items():
built_steps = []
for raw_step in raw_w["steps"]:
step_name = raw_step.pop("step")
step_ty = Steps.named(step_name)
munged = {
k.replace("-", "_"): v
for k, v in raw_step.items()
}
try:
step = step_ty(**munged)
except TypeError as e:
a = " ".join([f"{k}={v}" for k, v in munged.items()])
raise errors.BadSpec(
f"Unable to construct {step_name} with args {a}: {e}"
)
built_steps.append(step)
env = raw_w.get("env")
if not isinstance(env, dict) and env is not None:
raise errors.BadSpec(
f"Workflow {workflow_name} has wrong type for env: "
f"expected mapping, got {type(env).__name__}: {env}", )
# ensure that integers (e.g. ports) are treated as env vars
if isinstance(env, dict):
env = {k: str(v) for k, v in env.items()}
self.workflows[workflow_name] = Workflow(
workflow_name,
built_steps,
env=env,
composition=self,
)
# Resolve all services that reference an `mzbuild` image to a specific
# `image` reference.
for config in compose["services"].values():
if "mzbuild" in config:
image_name = config["mzbuild"]
if image_name not in self.repo.images:
raise errors.BadSpec(
f"mzcompose: unknown image {image_name}")
image = self.repo.images[image_name]
override_tag = os.getenv(f"MZBUILD_{image.env_var_name()}_TAG",
default_tag)
if override_tag is not None:
config["image"] = image.docker_name(override_tag)
print(
f"mzcompose: warning: overriding {image_name} image to tag {override_tag}",
file=sys.stderr,
)
del config["mzbuild"]
else:
self.images.append(image)
if "propagate-uid-gid" in config:
config["user"] = f"{os.getuid()}:{os.getgid()}"
del config["propagate-uid-gid"]
deps = self.repo.resolve_dependencies(self.images)
for config in compose["services"].values():
if "mzbuild" in config:
config["image"] = deps[config["mzbuild"]].spec()
del config["mzbuild"]
# Emit the munged configuration to a temporary file so that we can later
# pass it to Docker Compose.
tempfile = TemporaryFile()
os.set_inheritable(tempfile.fileno(), True)
yaml.dump(compose, tempfile, encoding="utf-8") # type: ignore
tempfile.flush()
self.file = tempfile
def __init__(self,
mock_class,
root_call,
args,
bufsize=0,
executable=None,
stdin=None,
stdout=None,
stderr=None,
preexec_fn=None,
close_fds=False,
shell=False,
cwd=None,
env=None,
universal_newlines=False,
startupinfo=None,
creationflags=0,
restore_signals=True,
start_new_session=False,
pass_fds=(),
encoding=None,
errors=None,
text=None):
self.mock = Mock()
self.class_instance_mock = mock_class.mock.Popen_instance
#: A :func:`unittest.mock.call` representing the call made to instantiate
#: this mock process.
self.root_call = root_call
#: The calls made on this mock process, represented using
#: :func:`~unittest.mock.call` instances.
self.calls = []
self.all_calls = mock_class.all_calls
cmd = shell_join(args)
behaviour = mock_class.commands.get(cmd, mock_class.default_behaviour)
if behaviour is None:
raise KeyError('Nothing specified for command %r' % cmd)
if callable(behaviour):
behaviour = behaviour(command=cmd, stdin=stdin)
self.behaviour = behaviour
stdout_value = behaviour.stdout
stderr_value = behaviour.stderr
if stderr == STDOUT:
line_iterator = chain.from_iterable(
zip_longest(stdout_value.splitlines(True),
stderr_value.splitlines(True)))
stdout_value = b''.join(l for l in line_iterator if l)
stderr_value = None
self.poll_count = behaviour.poll_count
for name, option, mock_value in (('stdout', stdout, stdout_value),
('stderr', stderr, stderr_value)):
value = None
if option is PIPE:
value = TemporaryFile()
value.write(mock_value)
value.flush()
value.seek(0)
if PY3 and (universal_newlines or text or encoding):
value = TextIOWrapper(value,
encoding=encoding,
errors=errors)
setattr(self, name, value)
if stdin == PIPE:
self.stdin = Mock()
for method in 'write', 'close':
record_writes = partial(self._record, ('stdin', method))
getattr(self.stdin, method).side_effect = record_writes
self.pid = behaviour.pid
#: The return code of this mock process.
self.returncode = None
if PY3:
self.args = args
def test_writestr_permissions(self):
for f in (TESTFN2, TemporaryFile(), StringIO()):
self.zip_test_writestr_permissions(f, zipfile.ZIP_STORED)