def parseGameMessage(self, parts):
if parts[0] == "game":
gameStateMessage = parts[1]
if gameStateMessage== 'round':
#update game round i
self.round = int(parts[2])
elif gameStateMessage == 'this_piece_type':
#update game this_piece_type s
self.this_piece_type = parts[2]
elif gameStateMessage == 'next_piece_type':
#update game next_piece_type s
self.next_piece_type = parts[2]
elif gameStateMessage == 'this_piece_position':
#update game this_piece_position i,i
position = parts[2].split(',')
self.this_piece_position["x"] = int(position[0])
self.this_piece_position["y"] = int(position[1])
else:
stderr.write('Unknown gameStateMessage: %s\n' % (parts[1]))
stderr.flush()
elif parts[0] == self.settings.bots["me"]:
self.players["me"].parsePlayerState(parts[1:])
elif parts[0] == self.settings.bots["opponent"]:
self.players["opponent"].parsePlayerState(parts[1:])
else:
stderr.write('Unknown gameStateMessage: %s\n' % (parts[0]))
stderr.flush()
def log_error(self, message, level=LOG_ERR):
"""Logging method with the same functionality like in Request object.
But as get_options read configuration from os.environ which could
not work in same wsgi servers like Apaches mod_wsgi.
This method write to stderr so messages, could not be found in
servers error log!
"""
if self.__log_level[0] >= level[0]:
if _unicode_exist and isinstance(message, unicode):
message = message.encode('utf-8')
try:
stderr.write("<%s> [%s] %s\n" % (level[1], self.__name,
message))
except UnicodeEncodeError:
if _unicode_exist:
message = message.decode('utf-8').encode(
'ascii', 'backslashreplace')
else:
message = message.encode(
'ascii', 'backslashreplace').decode('ascii')
stderr.write("<%s> [%s] %s\n" % (level[1], self.__name,
message))
stderr.flush()
def main(args):
try:
args.files = list(set(args.files))
for file_path in args.files:
if not file_path:
continue
if args.verbose:
print("Processing: {0}".format(file_path))
try:
with open(file_path, "rb") as input_file:
pdf_data = input_file.read()
except IOError as e:
stderr.write("{0}: {1}\n".format(file_path, e.strerror))
stderr.flush()
continue
# Backup the file with a different name
if not args.no_backup:
if args.verbose:
print("Creating backup: {0}.OLD".format(file_path))
shutil.move(file_path, "{0}.OLD".format(file_path))
# Modify the PDF file
new_pdf_data = remove_evil_links(pdf_data)
# Save the new file
with open(file_path, "wb") as out_file:
out_file.write(new_pdf_data)
if args.verbose:
print("Saving modified file: {0}".format(file_path))
except KeyboardInterrupt:
raise
def main(args):
napalm = {"name": "Napalm", "func": Napalm, "flag_set": args.napalm}
mamont = {"name": "Mamont", "func": Mamont, "flag_set": args.mamont}
filewatcher = {"name": "FileWatcher", "func": Filewatcher, "flag_set": args.filewatcher}
filemare = {"name": "FileMare", "func": Filemare, "flag_set": args.filemare}
custom_functions = []
for routine in (napalm, mamont, filewatcher, filemare):
if(routine["flag_set"]):
custom_functions.append(routine)
# Process -fw, -fm, -ma, -na flags if they are set
if(len(custom_functions) > 0):
functions = custom_functions
else:
functions = (napalm, mamont, filewatcher, filemare)
# Start the scraping process
for function in functions:
try:
stderr.write("\t-=[ {0} ]=-\n".format(function["name"]))
stderr.flush()
function["func"](args).search()
except(KeyboardInterrupt, EOFError):
continue
stderr.write("\n")
stderr.flush()
def daemonize(self):
"""
Forks the process(es) from the controlling terminal
and redirects I/O streams for logging.
"""
self.fork()
chdir(getcwd())
setsid()
umask(0)
self.fork()
stdout.flush()
stderr.flush()
si= file(self.stdin, 'w+')
so= file(self.stdout, 'a+')
se= file(self.stderr, 'a+', 0)
dup2(si.fileno(), stdin.fileno())
dup2(so.fileno(), stdout.fileno())
dup2(se.fileno(), stderr.fileno())
register(self.del_pid)
self.set_pid()
def _album(self, source):
""" Returns a list of image files from the passed source on success. """
try:
# Obtain the album image objects from the json data which is found
# in the passed source
if "<h2>All Categories</h2>" in source or "<a id=\"images\"" in source:
j = self._get_var_collectionData(source, "search")
else:
j = self._get_var_collectionData(source)
if not j:
raise EOFError
images = j.get("items").get("objects")
if not images:
raise EOFError
except(EOFError):
return "End of album"
# Try to detect the first page and print the estimated file count
# to stderr.
if(j["pageNumber"] == 1):
self._print_album_stats(source)
stderr.flush()
image_objects = []
for obj in images:
new_link = obj.get("fullsizeUrl")
up = urlparse(new_link)
new_link = "{0}~original".format(up.geturl())
obj["originalUrl"] = new_link
image_objects.append(ImageInfo(obj["name"], **obj))
image_objects = [ImageInfo(obj["name"], **obj) for obj in images if obj]
return image_objects
def benchmark(args, commit, commits):
# print progress indicator
stderr.write("\r{}/{} {}".format(commits.index(commit), len(commits), commit))
stderr.flush()
# checkout the given commit
run_from_repo(args, ["git", "checkout", "--force", commit])
try:
# run the benchmarks and store results as a stream
stream = StringIO(run_benchmarks(args))
except CalledProcessError as e:
# failure likely means a commit from before the benchmarks existed
return {}
# decode stream as CSV table
benchmarks = benchmarks_from_buffer(stream)
# refine table
report = table_from_benchmarks(benchmarks)
# determine the columns to extract
title_row = report[0]
name_index = title_row.index('name')
cpu_time_index = title_row.index('cpu_time')
# return list of tuples of benchmark test name to result
return {cell[name_index]: cell[cpu_time_index] for cell in report[1:]}
def _get_output_dir(self):
""" Returns the output directory, either the pwd or the directory
defined in the passed arguments.
Subalbums are given a subdirectory to store their images in.
"""
out = self._args.output_directory
if not out:
# Define the present working directory if it wasn't passed explicitly
# with the -o/--output-directory argument.
out = os.path.join(os.getcwd(), 'photobucket')
elif out.startswith("~"):
# Resolve the tilde char (which is the home directory on *nix) to
# it's actual destination.
home = os.environ.get("HOME")
if not home:
out = os.getcwd()
else:
out = os.path.join(home, out[1:])
if not os.path.isdir(out) and not os.path.isfile(out):
try:
os.makedirs(out)
except(OSError, IOError):
stderr.write("Failed to create output directory,",\
"does it already exist?\n")
stderr.flush()
exit(1)
# Add a trailing slash (or backslash) to the download directory, this is
# necessary otherwise we would get an error when trying to write the down-
# loaded file to the directory. (we want to write to file - not to the
# directory itself)
if not out.endswith(os.sep):
out += os.sep
return out
def _print_progress(self, iteration, n_iter,
cost=None, time_interval=10):
if self.print_progress > 0:
s = '\rIteration: %d/%d' % (iteration, n_iter)
if cost:
s += ' | Cost %.2f' % cost
if self.print_progress > 1:
if not hasattr(self, 'ela_str_'):
self.ela_str_ = '00:00:00'
if not iteration % time_interval:
ela_sec = time() - self._init_time
self.ela_str_ = self._to_hhmmss(ela_sec)
s += ' | Elapsed: %s' % self.ela_str_
if self.print_progress > 2:
if not hasattr(self, 'eta_str_'):
self.eta_str_ = '00:00:00'
if not iteration % time_interval:
eta_sec = ((ela_sec / float(iteration)) *
n_iter - ela_sec)
if eta_sec < 0.0:
eta_sec = 0.0
self.eta_str_ = self._to_hhmmss(eta_sec)
s += ' | ETA: %s' % self.eta_str_
stderr.write(s)
stderr.flush()
def java_encrypted_get(self, object_name):
tmpfile = NamedTemporaryFile()
args = " ".join([
"--bucket hello --object " + object_name,
"--kmsKeyId " + kms_key_id,
"--intercept http://127.0.0.1:%s/" % self.s3_server.port,
"--outputFile " + tmpfile.name,
"--read"
])
cwd=dirname(dirname(__file__)) + "/javacompat"
proc = Popen(["mvn", "-e", "exec:java",
"-Dexec.mainClass=org.kanga.dist.S3Test",
"-Dexec.args=" + args],
stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd=cwd)
out, err = proc.communicate()
if proc.returncode != 0:
stdout.write(out)
stderr.write(err)
stdout.flush()
stderr.flush()
raise ValueError("Get failed with exitcode %d" % proc.returncode)
tmpfile.seek(0)
data = tmpfile.read()
return data
def main():
'''
Die Parameter werden beim Aufruf aus dem config-Files abgerufen
usage: >> miniTopSim.py <ConfigFiles...>
'''
if(len(argv) > 1 ):
configFileNames = argv[1:]
else:
stderr.write('Error: usage: '+ argv[0] + ' <filename.cfg>')
stderr.flush()
exit(2)
# Read the parameter file
par.init()
print('Configuration files:\n{0}\n'.format(configFileNames))
for configFiles in configFileNames:
par.read(configFiles)
surface = Surface()
with open(surface.get_surfaceFile(), "w") as file:
#initial values
surface.write(file, 0)
total_time = par.TOTAL_TIME
dt = par.TIME_STEP
time = 1 #np.arange(initialTime, total_time + dt, dt)
while time <= total_time:
surface.process(dt)
surface.write(file, time * dt)
time += dt
plot = Plot(surface.get_surfaceFile())
plot.plot()
def _encoder_transform(X_s,layers,batch_range):
"""
Parameters:
----------
X_s: input data
layers: neuron layers (input shape + hidden layers)
batch_range: size of minibatch
Returs:
Input data with fetures as most latent representation
"""
ae= autoencoder(dimensions=layers)
learning_rate = 0.001
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(ae['cost'])
sess = tf.Session()
sess.run(tf.initialize_all_variables())
n_epoch=100
for epoch_i in range(n_epoch):
for start, end in zip(range(0, len(X_s), batch_range),range(batch_range, len(X_s), batch_range)):
input_ = X_s[start:end]
sess.run(optimizer, feed_dict={ae['x']: input_, ae['corrupt_prob']: [1.0]})
s="\r Epoch: %d Cost: %f"%(epoch_i, sess.run(ae['cost'],
feed_dict={ae['x']: X_s, ae['corrupt_prob']: [1.0]}))
stderr.write(s)
stderr.flush()
Z_0 = sess.run(ae['z'], feed_dict={ae['x']: X_s, ae['corrupt_prob']: [0.0]})
sess.close()
return Z_0
def task_message(task_id, escience_token, server_url, wait_timer, task='not_progress_bar'):
"""
Function to check create and destroy celery tasks running from orka-CLI
and log task state messages.
"""
payload = {"job": {"task_id": task_id}}
yarn_cluster_logger = ClusterRequest(escience_token, server_url, payload, action='job')
previous_response = {'job': {'state': 'placeholder'}}
response = yarn_cluster_logger.retrieve()
while 'state' in response['job']:
if response['job']['state'].replace('\r','') != previous_response['job']['state'].replace('\r',''):
if task == 'has_progress_bar':
stderr.write(u'{0}\r'.format(response['job']['state']))
stderr.flush()
else:
stderr.write('{0}'.format('\r'))
logging.log(SUMMARY, '{0}'.format(response['job']['state']))
previous_response = response
else:
stderr.write('{0}'.format('.'))
sleep(wait_timer)
response = yarn_cluster_logger.retrieve()
stderr.flush()
if 'success' in response['job']:
stderr.write('{0}'.format('\r'))
return response['job']['success']
elif 'error' in response['job']:
stderr.write('{0}'.format('\r'))
logging.error(response['job']['error'])
exit(error_fatal)
def parseAdvisoryString(wwaString):
advisoryString = ''
words = wwaString.split('^')
for word in words:
if '<None>' in word:
continue
entries = word.split('.')
hazard = entries[0]
advisory = entries[1]
if hazard in DEFS['wwa']['hazards']:
advisoryString += DEFS['wwa']['hazards'][hazard] + ' '
else:
stderr.write('WARNING: Unknown hazard code: ' + hazard + '\n'); stderr.flush()
if advisory in DEFS['wwa']['advisories']:
advisoryString += DEFS['wwa']['advisories'][advisory] + '\n'
else:
stderr.write('WARNING: Unknown advisory code: ' + advisory + '\n'); stderr.flush()
if len(advisoryString) == 0:
advisoryString = '<None>'
else:
advisoryString = advisoryString.strip().title()
return advisoryString
def _daemonize(self):
try:
pid = os.fork()
if pid > 0:
exit()
except OSError as e:
error(_('Error entering daemon mode: %s') % e.strerror)
exit()
os.chdir('/')
os.setsid()
os.umask(0)
stdout.flush()
stderr.flush()
si = open(os.devnull, 'r')
so = open(os.devnull, 'a+')
se = open(os.devnull, 'a+')
os.dup2(si.fileno(), stdin.fileno())
os.dup2(so.fileno(), stdout.fileno())
os.dup2(se.fileno(), stderr.fileno())
on_exit(self._quit)
old_log = getLogger()
if old_log.handlers:
for handler in old_log.handlers:
old_log.removeHandler(handler)
log(filename=self.logfile, level=self.loglevel,
format='%(asctime)s %(levelname)-8s %(message)s')
self._set_pid()
def _traceSuffixArray(operations, totalOperations):
if totalOperations==0:
percentage=100.
else:
percentage=float((operations*100)/totalOperations)
print >> _stderr, "Construction %.2f%% (%i/%i)\r"%(percentage,operations, totalOperations),
_stderr.flush()
def print_debug(msg):
msg = red(msg)
if print_method:
print_method(msg)
else:
stderr.write("{0}\n".format(msg))
stderr.flush()
def progresscontext(*args):
stderr.writelines(args)
stderr.flush()
try:
yield
finally:
print(file=stderr)
def doCombination(self):
## Contrary to Number-counting models, here each channel PDF already contains the nuisances
## So we just have to build the combined pdf
if len(self.DC.bins) > 1 or not self.options.forceNonSimPdf:
for (postfixIn,postfixOut) in [ ("","_s"), ("_bonly","_b") ]:
simPdf = ROOT.RooSimultaneous("model"+postfixOut, "model"+postfixOut, self.out.binCat) if self.options.noOptimizePdf else ROOT.RooSimultaneousOpt("model"+postfixOut, "model"+postfixOut, self.out.binCat)
for b in self.DC.bins:
pdfi = self.out.pdf("pdf_bin%s%s" % (b,postfixIn))
simPdf.addPdf(pdfi, b)
if len(self.DC.systs) and (not self.options.noOptimizePdf) and self.options.moreOptimizeSimPdf:
simPdf.addExtraConstraints(self.out.nuisPdfs)
if self.options.verbose:
stderr.write("Importing combined pdf %s\n" % simPdf.GetName()); stderr.flush()
self.out._import(simPdf)
if self.options.noBOnly: break
else:
self.out._import(self.out.pdf("pdf_bin%s" % self.DC.bins[0]).clone("model_s"), ROOT.RooFit.Silence())
if not self.options.noBOnly:
self.out._import(self.out.pdf("pdf_bin%s_bonly" % self.DC.bins[0]).clone("model_b"), ROOT.RooFit.Silence())
if self.options.fixpars:
pars = self.out.pdf("model_s").getParameters(self.out.obs)
iter = pars.createIterator()
while True:
arg = iter.Next()
if arg == None: break;
if arg.InheritsFrom("RooRealVar") and arg.GetName() != "r":
arg.setConstant(True);
def raise_error(self, message, section=SCRAPER, log_it=True):
message = "Error: " + message + ".\n"
if self.config["debug"] or not section == self.SCRAPER:
stderr.write(message)
stderr.flush()
if log_it:
try:
self.log.log_this(message, section)
except IOError:
self.issue_warning(self.GENERIC_FILE_IO_ERROR % {
"path_to_file" : self.log.current_path_to_log
}, self.INTERNAL)
except OSError:
self.issue_warning(self.CANNOT_CREATE_DIRECTORY % {
"directory" : self.log.current_path_to_log
}, self.INTERNAL)
exit(1)
def _load_links(self):
""" Sorts supported and not supported links in two lists and returns them. """
try: # Process the arguments, either read links from file or directly
# from the command line with the -u/--url flag.
if self.args.file:
with open(self.args.file) as file_:
links = [url.decode("utf8").strip() for url in file_ if url.strip()]
else:
links = [url.decode("utf8").strip() for url in self.args.url if url.strip()]
# Remove duplicates
links = list(set(links))
except IOError:
stderr.write("Couldn't open input file, are you sure the path is correct?\n")
stderr.flush()
exit(1)
not_supported = []
supported = []
for link in links:
parse = urlparse.urlparse(link)
if(not parse.scheme):
link = "http://{0}".format(link)
if(self.is_supported(link) and parse.path):
supported.append(link)
else:
not_supported.append(link)
return supported, not_supported
def stats(self):
""" Prints the amount of supported and not supported links to stderr. """
stderr.write("\tSupported links: {0}\n".format(len(self.supported_links)))
stderr.write("\tNot supported links: {0}\n".format(len(self.not_supported_links)))
total_links = len(self.supported_links) + len(self.not_supported_links)
stderr.write("{0}-( A total of {1} links )-{0}\n".format("-"*7, total_links))
stderr.flush()
def init():
"""Initialize hitch in this directory."""
if call(["which", "virtualenv"], stdout=PIPE):
stderr.write("You must have python-virtualenv installed to use hitch.\n")
stderr.flush()
exit(1)
if hitchdir.hitch_exists():
stderr.write("Hitch has already been initialized in this directory.\n")
stderr.flush()
exit(1)
makedirs(".hitch")
pip = path.abspath(path.join(".hitch", "virtualenv", "bin", "pip"))
call(["virtualenv", ".hitch/virtualenv", "--no-site-packages"])
call([pip, "install", "-U", "pip"])
if path.exists("hitchreqs.txt"):
call([pip, "install", "-r", "hitchreqs.txt"])
else:
call([pip, "install", "hitchtest"])
pip_freeze = check_output([pip, "freeze"])
with open("hitchreqs.txt", "w") as hitchreqs_handle:
hitchreqs_handle.write(pip_freeze)
def _print_progress(self, epoch, cost=None, train_acc=None,
valid_acc=None, time_interval=10):
if self.print_progress > 0:
s = '\rEpoch: %d/%d' % (epoch, self.epochs)
if cost is not None:
s += ' | Cost %.2f' % cost
if train_acc is not None:
s += ' | TrainAcc %.2f' % train_acc
if valid_acc is not None:
s += ' | ValidAcc %.2f' % valid_acc
if self.print_progress > 1:
if not hasattr(self, 'ela_str_'):
self.ela_str_ = '00:00:00'
if not epoch % time_interval:
ela_sec = time() - self.init_time_
self.ela_str_ = self._to_hhmmss(ela_sec)
s += ' | Elapsed: %s' % self.ela_str_
if self.print_progress > 2:
if not hasattr(self, 'eta_str_'):
self.eta_str_ = '00:00:00'
if not epoch % time_interval:
eta_sec = ((ela_sec / float(epoch)) *
self.epochs - ela_sec)
self.eta_str_ = self._to_hhmmss(eta_sec)
s += ' | ETA: %s' % self.eta_str_
stderr.write(s)
stderr.flush()
def report(): #!cov-ignore - the call to _exit kills coven before it records anything.
'At process exit, if any test failures occured, print a summary message and force process to exit with status code 1.'
from os import _exit
if _utest_failure_count > 0:
_errL(f'\nutest ran: {_utest_test_count}; failed: {_utest_failure_count}')
_stderr.flush()
_exit(1) # raising SystemExit has no effect in an atexit handler as of 3.5.2.
def parse(outprefix, from_date, to_date=None, ep_refs=None):
with open('ep_votes.json') as infile:
infile.seek(1)
line = infile.readline().strip()
i = 0
while line:
if line in (',', ']'):
i += 1
try:
line = infile.readline().strip()
except:
break
continue
vote_data = loads(line)
vote_date = parser.parse(vote_data['ts'])
if vote_date < from_date\
or (to_date and vote_date > to_date)\
or (ep_refs and vote_data.get('epref') not in ep_refs):
i += 1
try:
line = infile.readline().strip()
except:
break
continue
stderr.write('.'); stderr.flush()
for vid,vote_type in enumerate(vote_types):
if vote_type not in vote_data:
continue
for groups in vote_data[vote_type]['groups']:
for vote in groups['votes']:
if type(vote) != dict:
continue
meps.setdefault(vote['id'], [set(),set(),set()])[vid].add(i)
i += 1
try:
line = infile.readline().strip()
except:
break
stderr.write('[parsing done]\n')
with open(outprefix+'_vote_counts.csv', 'w') as outfile:
outfile.write('mep\tvote_count\n')
for mep in meps:
outfile.write('{0}\t{1}\n'.format(mep, sum(len(x) for x in meps[mep])))
stderr.write('[vote counts done]\n')
with open(outprefix+'_same_votes.csv', 'w') as outfile:
outfile.write('mep1\tmep2\tsame_vote_count\n')
for mep1, mep2 in combinations(meps.keys(), 2):
same_votes = 0
for cid,categ in enumerate(meps[mep1]):
for vote in categ:
if vote in meps[mep2][cid]:
same_votes += 1
outfile.write('{0}\t{1}\t{2}\n'.format(mep1,mep2,same_votes))
stderr.write('[same votes done]\n')
def download_image(self, file_info):
""" Downloads the image defined inside the passed fileinfo object. """
if(file_info.media_type == "video" and not self._args.all_filetypes):
return
out = self._args.output_directory
if not out:
# Define the present working directory if it wasn't passed explicitly
# with the -o/--output-directory argument.
out = os.getcwd()
elif out.startswith("~"):
# Resolve the tilde char (which is the home directory on *nix) to
# it's actual destination.
home = os.environ.get("HOME")
if not home:
out = os.getcwd()
else:
out = out[1:]
out = os.path.join(home, out)
if not os.path.isdir(out) and not os.path.isfile(out):
try:
os.makedirs(out)
except(OSError):
stderr.write("Failed to create output directory, does it already exist?\n")
stderr.flush()
return
# Add a trailing slash (or backslash) to the download directory, this is
# necessary otherwise we would get an error when trying to write the down-
# loaded file to the directory. (we want to write to file - not to the directory itself)
if not out.endswith(os.sep):
out += os.sep
out = os.path.join(out, file_info.filename)
# Handle duplicate file names
unique = 1
new_out = out
while os.path.isfile(new_out):
# Store the file extension and add a number between the name and the
# extension, then rebuild the path and check if it exists, if it does
# the whole process is repeated until an unique file name was built.
file_extension = out[out.rindex("."):]
new_out = out[:out.rindex(".")] + "(" + str(unique) + ")"
new_out += file_extension
unique += 1
if(not os.path.isfile(new_out)):
out = new_out
# Fetch the url stored inside the fileinfo object and write the fetched
# data into a file with the filename which is also stored inside the object.
with open(out, "wb") as f:
req = requests.get(file_info.link, stream=True)
if req.status_code != requests.codes.ok:
return
for chunk in req.iter_content():
if chunk:
f.write(chunk)
self._downloaded_images += 1
def run(self):
'''
Main loop
Keeps running while begin fed data from stdin.
Writes output to stdout, remember to flush.
'''
while not stdin.closed:
try:
rawline = stdin.readline()
# End of file check
if len(rawline) == 0:
break
line = rawline.strip()
# Empty lines can be ignored
if len(line) == 0:
continue
parts = line.split()
command = parts[0].lower()
self.log.debug("INCOMING:\t {0}".format(line))
if command == 'settings' or command == 'match':
self.update_settings(parts[1], parts[2])
pass
elif command.startswith('player'):
self.update_game_state(parts[0], parts[1], parts[2])
pass
elif command == 'action':
totalsize = len(self.table.hand) + len(self.player.hand)
self.log.debug("ACTION: totalsize={0}".format(totalsize))
if self.log: # converting this to pretty hands is expensive enough to catch
self.log.debug(" Table: {0}".format(self.table.getHumanHand()))
self.log.debug(" Us: {0}".format(self.player.getHumanHand()))
back = None
if totalsize == 2:
back = self.turn(parts[2], "pre_flop") + '\n'
elif totalsize == 5:
back = self.turn(parts[2], "flop") + '\n'
elif totalsize == 6:
back = self.turn(parts[2], "turn") + '\n'
elif totalsize == 7:
back = self.turn(parts[2], "river") + '\n'
else:
#self.log.debug('Unknown stage!')
pass
self.log.debug("OUT: {0}\n".format(back))
stdout.write(back)
stdout.flush()
else:
stderr.write("Unknown command: %s\n".format(command))
self.log.debug("ERR: Unknown command: %s\n".format(command))
stderr.flush()
except EOFError:
return
def serve_forever(self): print "starting server on %s:%i" % (self.addr, self.port) while not self.abort: self.handle_request() stdout.flush() stderr.flush() print "server done"
def tick(self):
self.cur += 1
newPercent = (100 * self.cur) / self.total
if newPercent > self.curPercent:
self.curPercent = newPercent
stderr.write(str(self.curPercent) + "%")
stderr.write("\r")
stderr.flush()
def main():
if len(sys.argv) == 1 or sys.argv[1] not in ('new', 'resume'):
print(parse_tools.top_usage, file=stderr)
return
mode = sys.argv[1]
del sys.argv[1]
if mode == 'new':
opts = parse_tools.two_stage_parse(parse_tools.cold)
elif mode == 'resume':
opts = parse_tools.resume.parse_args()
opts.device = None
if not opts.disable_cuda and torch.cuda.is_available():
opts.device = torch.device('cuda')
else:
opts.device = torch.device('cpu')
ckpt_path = util.CheckpointPath(opts.ckpt_template)
# Construct model
if mode == 'new':
# Initialize model
pre_params = parse_tools.get_prefixed_items(vars(opts), 'pre_')
enc_params = parse_tools.get_prefixed_items(vars(opts), 'enc_')
bn_params = parse_tools.get_prefixed_items(vars(opts), 'bn_')
dec_params = parse_tools.get_prefixed_items(vars(opts), 'dec_')
# Initialize data
sample_catalog = D.parse_sample_catalog(opts.sam_file)
data = D.WavSlices(sample_catalog, pre_params['sample_rate'],
opts.frac_permutation_use, opts.requested_wav_buf_sz)
dec_params['n_speakers'] = data.num_speakers()
#with torch.autograd.set_detect_anomaly(True):
model = ae.AutoEncoder(pre_params, enc_params, bn_params, dec_params)
print('Initializing model parameters', file=stderr)
model.initialize_weights()
# Construct overall state
state = checkpoint.State(0, model, data)
else:
state = checkpoint.State()
state.load(opts.ckpt_file)
print('Restored model and data from {}'.format(opts.ckpt_file), file=stderr)
state.model.set_geometry(opts.n_sam_per_slice)
state.data.set_geometry(opts.n_batch, state.model.input_size,
state.model.output_size)
state.model.to(device=opts.device)
#total_bytes = 0
#for name, par in model.named_parameters():
# n_bytes = par.data.nelement() * par.data.element_size()
# total_bytes += n_bytes
# print(name, type(par.data), par.size(), n_bytes)
#print('total_bytes: ', total_bytes)
# Initialize optimizer
model_params = state.model.parameters()
metrics = ae.Metrics(state.model, None)
batch_gen = state.data.batch_slice_gen_fn()
#loss_fcn = state.model.loss_factory(state.data.batch_slice_gen_fn())
# Start training
print('Starting training...', file=stderr)
print("Step\tLoss\tAvgProbTarget\tPeakDist\tAvgMax", file=stderr)
stderr.flush()
learning_rates = dict(zip(opts.learning_rate_steps, opts.learning_rate_rates))
start_step = state.step
if start_step not in learning_rates:
ref_step = util.greatest_lower_bound(opts.learning_rate_steps, start_step)
metrics.optim = torch.optim.Adam(params=model_params,
lr=learning_rates[ref_step])
while state.step < opts.max_steps:
if state.step in learning_rates:
metrics.optim = torch.optim.Adam(params=model_params,
lr=learning_rates[state.step])
# do 'pip install --upgrade scipy' if you get 'FutureWarning: ...'
metrics.update(batch_gen)
loss = metrics.optim.step(metrics.loss)
avg_peak_dist = metrics.peak_dist()
avg_max = metrics.avg_max()
avg_prob_target = metrics.avg_prob_target()
# Progress reporting
if state.step % opts.progress_interval == 0:
fmt = "{}\t{:.5f}\t{:.5f}\t{:.5f}\t{:.5f}"
print(fmt.format(state.step, loss, avg_prob_target, avg_peak_dist,
avg_max), file=stderr)
stderr.flush()
# Checkpointing
if state.step % opts.save_interval == 0 and state.step != start_step:
ckpt_file = ckpt_path.path(state.step)
state.save(ckpt_file)
print('Saved checkpoint to {}'.format(ckpt_file), file=stderr)
state.step += 1
def main(arguments=None):
args = arguments or argv[1:]
fileparser = None
argparser = argparse.ArgumentParser(
description='Convert pep8 or flake8 output to HTML',
prog=NAME,
epilog=
'%(name)s accepts input either from stdin or from a filename argument.\n'
+
'Unless specified otherwise with -o OUTPUT_FILE, %(name)s outputs to stdout.'
% {'name': NAME})
argparser.add_argument(
'filename',
nargs='?',
type=str,
help='Path to file containing pep8 or flake8 results.')
argparser.add_argument(
'-v',
'--verbose',
action='store_true',
help='Enable verbose output (only if --output-file is specified)')
argparser.add_argument('--version',
action='store_true',
help='Prints %s version and exists' % NAME)
argparser.add_argument(
'-o',
'--output-file',
type=str,
help=
'Outputs the HTML data to the specified file and enables the use of the --verbose option.'
)
argparser.add_argument('-g',
'--generator',
choices=GENERATOR_CHOICES.keys(),
help='Selects the generator Html or TeamCity')
argparser.add_argument('-r',
'--report-name',
type=str,
default=DEFAULT_REPORT_NAME,
help='Name for the report.')
# Fetch the provided arguments from sys.argv
args = argparser.parse_args(args)
if args.version:
print('%s version %s' % (NAME, VERSION))
exit(0)
if args.filename:
try:
f = open(args.filename)
fileparser = Parser(f)
except IOError as e:
stderr.write('Could not open file: %s' % e)
stderr.flush()
exit(1)
else:
# We need to check if stdin is piped or read from file, since we dont want
# stdin to hang at terminal input
mode = fstat(0).st_mode
if S_ISFIFO(mode) or S_ISREG(mode):
fileparser = Parser(stdin)
else:
# stdin is terminal input at this point
argparser.print_help()
exit(0)
# Generate the HTML report to output_file if not None, else print to stdout
generator = GeneratorBase.create_generator(args.generator, fileparser,
args.report_name)
if generator is None:
stderr.write('Unsupported generator: %s' % args.generator)
stderr.flush()
exit(1)
generator.analyze(output_file=args.output_file)
def _err(msg: str) -> None:
stderr.write(msg)
stderr.flush()
def log(message):
stderr.write(message + "\n")
stderr.flush()
def display(self, info):
stderr.write('\r ')
stderr.write(f'\r [{self.elapsed_seconds}s] {info}')
stderr.flush()
def debug_msg(self, msg):
""" Write msg to the debug stream """
if self._debug_mode:
stderr.write(msg)
stderr.flush()
def announce_test(name):
stderr.write("%s%s%s: " % (BOLD, name, NORMAL))
stderr.flush()
def GET(uid=None, **kwargs):
"""Download the tar file created by the cart."""
if not uid:
cherrypy.response.headers['Content-Type'] = 'application/json'
return bytes_type(dumps({'message': 'Pacifica Cartd Interface Up and Running'}))
rtn_name = kwargs.get(
'filename', 'data_' + datetime.now().strftime('%Y_%m_%d_%H_%M_%S') + '.tar')
# get the bundle path if available
cart_utils = Cartutils()
Cart.database_connect()
cart_path = cart_utils.available_cart(uid)
Cart.database_close()
if cart_path is False:
# cart not ready
cherrypy.response.status = '202 Accepted'
return bytes_type('The cart is not ready for download.')
elif cart_path is None:
# cart not found
raise cherrypy.HTTPError(
404, 'The cart does not exist or has already been deleted')
if os.path.isdir(cart_path):
# give back bundle here
stderr.flush()
# want to stream the tar file out
(rpipe, wpipe) = os.pipe()
rfd = os.fdopen(rpipe, 'rb')
wfd = os.fdopen(wpipe, 'wb')
def do_work():
"""The child thread writes the data to the pipe."""
mytar = TarFile.open(fileobj=wfd, mode='w|')
mytar.add(cart_path, arcname=rtn_name.replace('.tar', ''))
mytar.close()
wfd.close()
# open the pipe as a file
wthread = Thread(target=do_work)
wthread.daemon = True
wthread.start()
cherrypy.response.stream = True
cherrypy.response.headers['Content-Type'] = 'application/octet-stream'
cherrypy.response.headers['Content-Disposition'] = 'attachment; filename={}'.format(
rtn_name)
xfer_size = parse_size(get_config().get('cartd', 'transfer_size'))
def read():
"""read some size from rfd."""
buf = rfd.read(xfer_size)
while buf:
yield buf
buf = rfd.read(xfer_size)
wthread.join()
return read()
elif os.path.isfile(cart_path):
return static.serve_file(
cart_path,
'application/octet-stream',
'attachment',
rtn_name
)
raise cherrypy.HTTPError(404, 'Not Found')
def main():
logging.basicConfig(stream=stderr, level=INFO)
a = ArgumentParser()
a.add_argument('-data', dest='data', required=True, metavar='WORDLIST',
help="a text file (the corpus) consisting of one word per line. The word may be preceded by a word"\
" count (separated by whitespace), otherwise a count of one is assumed. If the same word "\
"occurs many times, the counts are accumulated.")
a.add_argument('-finish', dest='finish', metavar='float', type=float, default=0.005,
help="convergence threshold. From one pass over all input words to the next, "\
"if the overall coding length in bits (i.e. logprob) of the lexicon together with the corpus "\
"improves less than this value times the number of word types (distinct word forms) in the "\
"data, the program stops. (If this value is small the program runs for a longer time and the "\
"result is in principle more accurate. However, the changes in word splittings during the "\
"last training epochs are usually very small.) The value must be within the range: 0 < float "\
"< 1. Default 0.005")
a.add_argument(
'-rand',
dest='rand',
metavar='int',
type=int,
default=0,
help=
"random seed that affects the sorting of words when processing them. Default 0"
)
a.add_argument('-gammalendistr', dest='gammalendistr', type=float, metavar='float', nargs=2,
help="Use Gamma Length distribution with two parameters. Float1 is the prior for the most common "\
"morph length in the lexicon, such that 0 < float1 <= 24*float2. Float2 is the beta value of "\
"the Gamma pdf, such that beta > 0. The beta value affects the wideness of the morph length "\
"distribution. The higher beta, the wider and less discriminative the distribution. If this "\
"option is omitted, morphs in the lexicon are terminated with an end-of-morph character, "\
"which corresponds to an exponential pdf for morph lengths. Suggested values: float1 = 7.0, "\
"float2 = 1.0 ")
a.add_argument('-zipffreqdistr', dest='zipffreqdistr', type=float, metavar='float',
help="Use Zipf Frequency distribution with paramter float1 for the proportion of morphs in the "\
"lexicon that occur only once in the data (hapax legomena): 0 < value < 1. If this option is "\
"omitted a (non-informative) morph frequency distribution based on enumerative coding is used"\
" instead. Suggested value: 0.5")
a.add_argument('-load', dest='load', metavar='filename',
help="An existing model for word splitting is loaded from a file (which is the output of an "\
"earlier run of this program) and the words in the corpus defined using the option '-data "\
"wordlist' are segmented according to the loaded model. That is, "\
"no learning of a new model takes place. The existing model is simply used for segmenting a " \
"list of words. The segmentation takes place using Viterbi search. No new morphs are ever " \
"created (except one-letter morphs, if there is no other way of segmenting a particular input" \
" word)")
a.add_argument('-encoding',
dest='encoding',
help='Input encoding (defaults to local encoding)')
a.add_argument('-savememory', type=int, nargs='?', help=SUPPRESS)
options = a.parse_args()
if options.load is not None:
m = MorphModel(vars(options))
m.load(options.load)
for word in open(options.data):
print(' + '.join(m.viterbi_segment_word(word.strip())))
else:
m = MorphModel(vars(options))
m.train(options.data)
stderr.flush()
m.print_segmentation()
def container_dispersion_report(coropool, connpool, account, container_ring,
retries, output_missing_partitions, policy):
with connpool.item() as conn:
containers = [
c['name']
for c in conn.get_account(prefix='dispersion_%d' % policy.idx,
full_listing=True)[1]
]
containers_listed = len(containers)
if not containers_listed:
print(
'No containers to query. Has '
'swift-dispersion-populate been run?',
file=stderr)
stderr.flush()
return
retries_done = [0]
containers_queried = [0]
container_copies_missing = defaultdict(int)
container_copies_found = [0]
container_copies_expected = [0]
begun = time()
next_report = [time() + 2]
def direct(container, part, nodes):
found_count = 0
for node in nodes:
error_log = get_error_log('%(ip)s:%(port)s/%(device)s' % node)
try:
attempts, _junk = direct_client.retry(
direct_client.direct_head_container,
node,
part,
account,
container,
error_log=error_log,
retries=retries)
retries_done[0] += attempts - 1
found_count += 1
except ClientException as err:
if err.http_status not in (404, 507):
error_log('Giving up on /%s/%s/%s: %s' %
(part, account, container, err))
except (Exception, Timeout) as err:
error_log('Giving up on /%s/%s/%s: %s' %
(part, account, container, err))
if output_missing_partitions and \
found_count < len(nodes):
missing = len(nodes) - found_count
print('\r\x1B[K', end='')
stdout.flush()
print('# Container partition %s missing %s cop%s' %
(part, missing, 'y' if missing == 1 else 'ies'),
file=stderr)
container_copies_found[0] += found_count
containers_queried[0] += 1
container_copies_missing[len(nodes) - found_count] += 1
if time() >= next_report[0]:
next_report[0] = time() + 5
eta, eta_unit = compute_eta(begun, containers_queried[0],
containers_listed)
if not json_output:
print('\r\x1B[KQuerying containers: %d of %d, %d%s left, %d '
'retries' % (containers_queried[0], containers_listed,
round(eta), eta_unit, retries_done[0]),
end='')
stdout.flush()
container_parts = {}
for container in containers:
part, nodes = container_ring.get_nodes(account, container)
if part not in container_parts:
container_copies_expected[0] += len(nodes)
container_parts[part] = part
coropool.spawn(direct, container, part, nodes)
coropool.waitall()
distinct_partitions = len(container_parts)
copies_found = container_copies_found[0]
copies_expected = container_copies_expected[0]
value = 100.0 * copies_found / copies_expected
elapsed, elapsed_unit = get_time_units(time() - begun)
container_copies_missing.pop(0, None)
if not json_output:
print(
'\r\x1B[KQueried %d containers for dispersion reporting, '
'%d%s, %d retries' %
(containers_listed, round(elapsed), elapsed_unit, retries_done[0]))
if containers_listed - distinct_partitions:
print('There were %d overlapping partitions' %
(containers_listed - distinct_partitions))
for missing_copies, num_parts in container_copies_missing.items():
print(
missing_string(num_parts, missing_copies,
container_ring.replica_count))
print('%.02f%% of container copies found (%d of %d)' %
(value, copies_found, copies_expected))
print('Sample represents %.02f%% of the container partition space' %
(100.0 * distinct_partitions / container_ring.partition_count))
stdout.flush()
return None
else:
results = {
'retries': retries_done[0],
'overlapping': containers_listed - distinct_partitions,
'pct_found': value,
'copies_found': copies_found,
'copies_expected': copies_expected
}
for missing_copies, num_parts in container_copies_missing.items():
results['missing_%d' % (missing_copies)] = num_parts
return results
def error(out):
stderr.write("\n ** Error: %s\n" % out)
stderr.flush()
def doIndividualModels(self):
if self.options.verbose:
stderr.write("Creating pdfs for individual modes (%d): " %
len(self.DC.bins))
stderr.flush()
for i, b in enumerate(self.DC.bins):
#print " + Getting model for bin %s" % (b)
pdfs = ROOT.RooArgList()
bgpdfs = ROOT.RooArgList()
coeffs = ROOT.RooArgList()
bgcoeffs = ROOT.RooArgList()
for p in self.DC.exp[b].keys(
): # so that we get only self.DC.processes contributing to this bin
if self.DC.exp[b][p] == 0: continue
if self.physics.getYieldScale(b, p) == 0:
continue # exclude really the pdf
#print " +--- Getting pdf for %s in bin %s" % (p,b)
(pdf,
coeff) = (self.getPdf(b, p),
self.out.function("n_exp_bin%s_proc_%s" % (b, p)))
if self.options.optimizeExistingTemplates:
pdf1 = self.optimizeExistingTemplates(pdf)
if (pdf1 != pdf):
self.out.dont_delete.append(pdf1)
pdf = pdf1
extranorm = self.getExtraNorm(b, p)
if extranorm:
prodset = ROOT.RooArgList(
self.out.function("n_exp_bin%s_proc_%s" % (b, p)))
for X in extranorm:
prodset.add(self.out.function(X))
prodfunc = ROOT.RooProduct(
"n_exp_final_bin%s_proc_%s" % (b, p), "", prodset)
self.out._import(prodfunc)
coeff = self.out.function("n_exp_final_bin%s_proc_%s" %
(b, p))
pdf.setStringAttribute("combine.process", p)
pdf.setStringAttribute("combine.channel", b)
pdf.setAttribute("combine.signal", self.DC.isSignal[p])
coeff.setStringAttribute("combine.process", p)
coeff.setStringAttribute("combine.channel", b)
coeff.setAttribute("combine.signal", self.DC.isSignal[p])
pdfs.add(pdf)
coeffs.add(coeff)
if not self.DC.isSignal[p]:
bgpdfs.add(pdf)
bgcoeffs.add(coeff)
if self.options.verbose > 1:
print "Creating RooAddPdf %s with %s elements" % (
"pdf_bin" + b, coeffs.getSize())
sum_s = ROOT.RooAddPdf("pdf_bin%s" % b, "", pdfs, coeffs)
if not self.options.noBOnly:
sum_b = ROOT.RooAddPdf("pdf_bin%s_bonly" % b, "", bgpdfs,
bgcoeffs)
if b in self.pdfModes:
sum_s.setAttribute('forceGen' + self.pdfModes[b].title())
if not self.options.noBOnly:
sum_b.setAttribute('forceGen' + self.pdfModes[b].title())
if len(self.DC.systs) and (self.options.noOptimizePdf
or not self.options.moreOptimizeSimPdf):
## rename the pdfs
sum_s.SetName("pdf_bin%s_nuis" % b)
if not self.options.noBOnly:
sum_b.SetName("pdf_bin%s_bonly_nuis" % b)
# now we multiply by all the nuisances, but avoiding nested products
# so we first make a list of all nuisances plus the RooAddPdf
sumPlusNuis_s = ROOT.RooArgList(self.out.nuisPdfs)
sumPlusNuis_s.add(sum_s)
# then make RooProdPdf and import it
pdf_s = ROOT.RooProdPdf("pdf_bin%s" % b, "", sumPlusNuis_s)
if not self.options.noBOnly:
sumPlusNuis_b = ROOT.RooArgList(self.out.nuisPdfs)
sumPlusNuis_b.add(sum_b)
pdf_b = ROOT.RooProdPdf("pdf_bin%s_bonly" % b, "",
sumPlusNuis_b)
if b in self.pdfModes:
pdf_s.setAttribute('forceGen' + self.pdfModes[b].title())
if not self.options.noBOnly:
pdf_b.setAttribute('forceGen' +
self.pdfModes[b].title())
if self.options.verbose:
if i > 0: stderr.write("\b\b\b\b\b")
stderr.write(". %4d" % (i + 1))
stderr.flush()
self.out._import(pdf_s, ROOT.RooFit.RenameConflictNodes(b))
if not self.options.noBOnly:
self.out._import(pdf_b, ROOT.RooFit.RecycleConflictNodes(),
ROOT.RooFit.Silence())
else:
if self.options.verbose:
if i > 0: stderr.write("\b\b\b\b\b")
stderr.write(". %4d" % (i + 1))
stderr.flush()
self.out._import(sum_s, ROOT.RooFit.RenameConflictNodes(b))
if not self.options.noBOnly:
self.out._import(sum_b, ROOT.RooFit.RecycleConflictNodes(),
ROOT.RooFit.Silence())
if self.options.verbose:
stderr.write("\b\b\b\bdone.\n")
stderr.flush()
greet = '{}, {}!'.format(*greet)
if case == 'upper':
greet = greet.upper()
elif case == 'lower':
greet = greet.lower()
if delay == 0:
print(greet, file=stderr)
else:
for x in greet:
stderr.write(x)
stderr.flush()
sleep(delay / 1000.0)
print(file=stderr)
else:
return HALT
else:
return HALT
if __name__ == '__main__':
cli = CommandLineMapper()
cli.add(hello)
def announce_result(result, info):
stderr.write(result)
if info != "":
stderr.write(": %s" % info)
stderr.write("\n")
stderr.flush()
def cmd(self, command, **kwargs):
# prepare the environ, based on the system + our own env
env = copy(environ)
env.update(self.environ)
# prepare the process
kwargs.setdefault('env', env)
kwargs.setdefault('stdout', PIPE)
kwargs.setdefault('stderr', PIPE)
kwargs.setdefault('close_fds', True)
kwargs.setdefault('shell', True)
kwargs.setdefault('show_output', self.log_level > 1)
show_output = kwargs.pop('show_output')
get_stdout = kwargs.pop('get_stdout', False)
get_stderr = kwargs.pop('get_stderr', False)
break_on_error = kwargs.pop('break_on_error', True)
sensible = kwargs.pop('sensible', False)
if not sensible:
self.debug('Run {0!r}'.format(command))
else:
if type(command) in (list, tuple):
self.debug('Run {0!r} ...'.format(command[0]))
else:
self.debug('Run {0!r} ...'.format(command.split()[0]))
self.debug('Cwd {}'.format(kwargs.get('cwd')))
# open the process
if sys.platform == 'win32':
kwargs.pop('close_fds', None)
process = Popen(command, **kwargs)
# prepare fds
fd_stdout = process.stdout.fileno()
fd_stderr = process.stderr.fileno()
if fcntl:
fcntl.fcntl(fd_stdout, fcntl.F_SETFL,
fcntl.fcntl(fd_stdout, fcntl.F_GETFL) | os.O_NONBLOCK)
fcntl.fcntl(fd_stderr, fcntl.F_SETFL,
fcntl.fcntl(fd_stderr, fcntl.F_GETFL) | os.O_NONBLOCK)
ret_stdout = [] if get_stdout else None
ret_stderr = [] if get_stderr else None
while True:
try:
readx = select.select([fd_stdout, fd_stderr], [], [])[0]
except select.error:
break
if fd_stdout in readx:
chunk = process.stdout.read()
if not chunk:
break
if get_stdout:
ret_stdout.append(chunk)
if show_output:
if IS_PY3:
stdout.write(chunk.decode('utf-8'))
else:
stdout.write(chunk)
if fd_stderr in readx:
chunk = process.stderr.read()
if not chunk:
break
if get_stderr:
ret_stderr.append(chunk)
if show_output:
if IS_PY3:
stderr.write(chunk.decode('utf-8'))
else:
stderr.write(chunk)
stdout.flush()
stderr.flush()
process.communicate()
if process.returncode != 0 and break_on_error:
self.error('Command failed: {0}'.format(command))
raise BuildozerCommandException()
if ret_stdout:
ret_stdout = b''.join(ret_stdout)
if ret_stderr:
ret_stderr = b''.join(ret_stderr)
return (ret_stdout.decode('utf-8', 'ignore') if ret_stdout else None,
ret_stderr.decode('utf-8') if ret_stderr else None,
process.returncode)
def log(msg):
stderr.write("%s\n" % msg)
stderr.flush()
def next_step(self):
stderr.write('\n')
stderr.flush()
def statusline(line):
"""Print status line if condition is true. Correctly clears the line before printing"""
stderr.write('\r\033[2K%s\r' % line)
stderr.flush()
def pipefork():
"""Context manager that forks with pipes between parent and child.
Use like so::
with pipefork() as (pid, fin, fout):
if pid == 0:
# This is the child.
...
else:
# This is the parent.
...
Pipes are set up so that the parent can write to the child, and
vice-versa.
In the child, ``fin`` is a file that reads from the parent, and ``fout``
is a file that writes to the parent.
In the parent, ``fin`` is a file that reads from the child, and ``fout``
is a file that writes to the child.
Be careful to think about closing these file objects to avoid deadlocks.
For example, the following will deadlock:
with pipefork() as (pid, fin, fout):
if pid == 0:
fin.read() # Read from the parent.
fout.write(b'Moien') # Greet the parent.
else:
fout.write(b'Hello') # Greet the child.
fin.read() # Read from the child *BLOCKS FOREVER*
The reason is that the read in the child never returns because the pipe is
never closed. Closing ``fout`` in the parent resolves the problem::
with pipefork() as (pid, fin, fout):
if pid == 0:
fin.read() # Read from the parent.
fout.write(b'Moien') # Greet the parent.
else:
fout.write(b'Hello') # Greet the child.
fout.close() # Close the write pipe to the child.
fin.read() # Read from the child.
Exceptions raised in the child are magically re-raised in the parent. When
the child has died for another reason, a signal perhaps, a `PipeForkError`
is raised with an explanatory message.
Signal handlers in the child are NOT modified. This means that signal
handlers set in the parent will still be present in the child.
:raises: `PipeForkError` when the child process dies a somewhat unnatural
death, e.g. by a signal or when writing a crash-dump fails.
"""
crashfile = TemporaryFile()
c2pread, c2pwrite = os.pipe()
p2cread, p2cwrite = os.pipe()
pid = os.fork()
if pid == 0:
# Child: this conditional branch runs in the child process.
try:
os.close(c2pread)
os.close(p2cwrite)
with os.fdopen(p2cread, 'rb') as fin:
with os.fdopen(c2pwrite, 'wb') as fout:
yield pid, fin, fout
stdout.flush()
stderr.flush()
except SystemExit as se:
# Exit hard, not soft.
os._exit(se.code)
except:
try:
# Pickle error to crash file.
pickle.dump(Failure(), crashfile, pickle.HIGHEST_PROTOCOL)
crashfile.flush()
finally:
# Exit hard.
os._exit(2)
finally:
# Exit hard.
os._exit(0)
else:
# Parent: this conditional branch runs in the parent process.
os.close(c2pwrite)
os.close(p2cread)
with os.fdopen(c2pread, 'rb') as fin:
with os.fdopen(p2cwrite, 'wb') as fout:
yield pid, fin, fout
# Wait for the child to finish.
_, status = os.waitpid(pid, 0)
signal = (status & 0xff)
code = (status >> 8) & 0xff
# Check for a saved crash.
crashfile.seek(0)
try:
error = pickle.load(crashfile)
except EOFError:
# No crash was recorded.
error = None
else:
# Raise exception from child.
error.raiseException()
finally:
crashfile.close()
if os.WIFSIGNALED(status):
# The child was killed by a signal.
raise PipeForkError(
"Child killed by signal %d (%s)" % (
signal, signal_names.get(signal, "?")))
elif code != 0:
# The child exited with a non-zero code.
raise PipeForkError(
"Child exited with code %d" % code)
else:
# All okay.
pass
def debug(msg, *args, **kwargs):
level = kwargs.pop('level', 1)
if level <= VERBOSE: # pragma: no cover
print('[pgctl] DEBUG:', msg % args, file=stderr)
stderr.flush()
def dbg(ostr):
stderr.write('dbg: %s' % ostr)
stderr.flush()
return
def get(self, env, start_response):
"""Download the tar file created by the cart"""
resp = cart_interface_responses.Responses()
rtn_name = None
if 'filename' in parse_qs(env['QUERY_STRING']):
rtn_name = os.path.basename(
parse_qs(env['QUERY_STRING'])['filename'][0])
else:
rtn_name = "data_" + datetime.now().strftime(
'%Y_%m_%d_%H_%M_%S') + ".tar"
uid = fix_cart_uid(env['PATH_INFO'])
is_valid = is_valid_uid(uid)
if not is_valid:
self._response = resp.invalid_uid_error_response(
start_response, uid)
return self.return_response()
#get the bundle path if available
cart_utils = Cartutils()
cart_path = cart_utils.available_cart(uid)
if cart_path is False:
#cart not ready
self._response = resp.unready_cart(start_response)
elif cart_path is None:
#cart not found
self._response = resp.cart_not_found(start_response)
else:
if os.path.isdir(cart_path):
#give back bundle here
stderr.flush()
try:
#want to stream the tar file out
(rpipe, wpipe) = os.pipe()
cpid = os.fork()
# the fork screws up coverage testing... :(
if cpid == 0: # pragma: no cover
# we are the child process
#write the data to the pipe
os.close(rpipe)
wfd = os.fdopen(wpipe, 'wb')
mytar = TarFile.open(fileobj=wfd, mode='w|')
mytar.add(cart_path,
arcname=rtn_name.replace('.tar', ''))
mytar.close()
#to exit from the fork child without killing the parent
#we need to use_exit. Disabling the pylint for this
#so that it doesnt throw an error
# pylint: disable=protected-access
os._exit(0)
# pylint: enable=protected-access
# we are the parent
os.close(wpipe)
#open the pipe as a file
rfd = os.fdopen(rpipe, 'rb')
start_response(
'200 OK',
[('Content-Type', 'application/octet-stream'),
('Content-Disposition',
'attachment; filename=' + str(rtn_name))])
if 'wsgi.file_wrapper' in env:
return env['wsgi.file_wrapper'](rfd, BLOCK_SIZE)
return iter(lambda: rfd.read(BLOCK_SIZE), '')
except IOError:
self._response = resp.bundle_doesnt_exist(start_response)
else:
self._response = resp.bundle_doesnt_exist(start_response)
return self.return_response()
return self.return_response()
def progress_msg(processed, total):
"""Update user on percent done"""
if total > 1:
percent = int((float(processed) / total) * 100)
stderr.write("\r[%d/%d] %d%%" % (processed, total, percent))
stderr.flush()
def __init__(self, chr, cp_data, starts, ends, cutoff_scale, **kwargs):
max_merge = kwargs.get("max_merge", 0.5)
use_means = kwargs.get("use_means", False)
n_scales = kwargs.get("n_scales", 51)
#n_scales=kwargs.get("n_scales",30)
scale_width = kwargs.get('scale_width', 1)
n_bin_smoothings = kwargs.get('-n_bin_smoothings', 0)
smoothing_kernel = kwargs.get('smoothing_kernel', np.array([1, 2, 1]))
self.chr = chr
self.cutoff_scale = cutoff_scale
self.scales = list(np.arange(1, n_scales, scale_width))
self.starts = starts
self.ends = ends
self.n_wnds = self.starts.shape[0]
self.cp_data = cp_data
self.der1 = np.zeros((len(self.scales), self.n_wnds), dtype=np.float32)
self.der2 = np.zeros((len(self.scales), self.n_wnds), dtype=np.int8)
self.vars = get_windowed_variance(cp_data.astype(np.float64), 500)
self.l_vars = np.roll(self.vars, 501)
self.r_vars = np.roll(self.vars, -501)
print("scales range from %f-%f" % (self.scales[0], self.scales[-1]),
file=stderr)
for i in range(n_bin_smoothings):
print("doing binomial smooth #%d" % i, file=stderr)
cp_data = ndi.convolve1d(
cp_data, smoothing_kernel) / np.sum(smoothing_kernel)
transitions_by_scale = {}
print("finding contours...", file=stderr)
for i_scale, scale in enumerate(self.scales):
stderr.write("%.2f " % (scale))
stderr.flush()
g1 = ndi.gaussian_filter1d(cp_data, scale, order=1)
g2 = ndi.gaussian_filter1d(cp_data, scale, order=2)
edges, pos_edges, neg_edges = self.get_n_edges(g1, g2)
self.der1[i_scale, :] = g1
self.der2[i_scale, :] = pos_edges - neg_edges
transitions_by_scale[scale] = (edges, pos_edges, neg_edges)
stderr.write("done\n")
self.contour_intersects, x_intercept_to_scale = get_contours(self.der2)
######NOW we have all the per-scale contours
#print contour_intersects
edges_passing_cutoff = []
curr_all_edges = []
curr_all_edges_scales = []
#take all the edges discovered at some scale
for scale, edges in self.contour_intersects.items():
curr_all_edges.extend(edges)
curr_all_edges_scales.extend([scale for i in range(len(edges))])
if scale >= cutoff_scale:
edges_passing_cutoff.extend(edges)
edges_passing_cutoff = sorted(set(edges_passing_cutoff))
all_edges_scales = sorted(zip(curr_all_edges, curr_all_edges_scales))
stderr.write("hierarchically merging segments\n")
t = time.time()
segments_s, segments_e, cps = c_hierarch_merge_edges(
cp_data, edges_passing_cutoff, max_merge, use_means, self.n_wnds,
self.starts, self.ends)
#segments_s, segments_e, cps = hierarch_merge_edges(cp_data,
# edges_passing_cutoff,
# max_merge,use_means)
self.segment_edges = (segments_s, segments_e, cps)
print("hierarchical clustering completed in %fs" % (time.time() - t),
file=stderr)
def solve(target,
piece_shapes,
multi=False,
just_count=False,
verbose=False,
default_guess=None):
"""
target is a shape.
piece_shapes is a dict of {label_letter: shape}.
"""
# 7 pieces, up to 27 target bloxels, up to about 700 piece-orientations.
# Let's only use "bloxel" to refer to points in the target.
state = State(verbose=verbose)
# First we set up the Constraints. Each is like a deputy who later
# gets assigned some variables and will make sure the number of True
# variables under his watch stays in a range.
# Each bloxel is occupied exactly once.
point_bloxels = dict((point, Bloxel(point)) for point in target)
bloxels = point_bloxels.values()
occupied_once = {}
for bloxel in bloxels:
occupied_once[bloxel] = BoolConstraint(state,
bloxel=bloxel,
min_True=1,
max_True=1)
# Each piece is used exactly once: to occupy a bloxel, or for nothing:
labeled_pieces = dict((label, Piece(label, shape))
for label, shape in piece_shapes.iteritems())
pieces = labeled_pieces.values()
oriented_one_way = {}
for piece in pieces:
oriented_one_way[piece] = BoolConstraint(state,
piece=piece,
min_True=1,
max_True=1)
# Constraints on how many pieces are unused, given sizes of pieces:
n_unused = get_n_unused(pieces, target)
t_unused = sum(n_unused.values())
if t_unused == 0:
print "All", len(pieces), "pieces used."
else:
print len(pieces) - t_unused, "pieces used."
# A fixed number of pieces of each size will be unused.
how_many_unused = {}
for piece_size in n_unused:
how_many_unused[piece_size] = \
BoolConstraint(state, piece_size=piece_size,
min_True=n_unused[piece_size],
max_True=n_unused[piece_size])
# Create the Variables and assign them to their Constraints.
for piece in pieces:
piece_unused = BoolVar(state, piece=piece, label="unused")
# Unused is one way a piece can be "oriented"; see loop below.
oriented_one_way[piece].constrain(piece_unused)
piece_size = len(piece.shape)
how_many_unused[piece_size].constrain(piece_unused)
for orientation in all_orientations_fitting(piece, target):
orient_bloxels = [point_bloxels[pt] for pt in orientation.shape]
piece_oriented_thus = BoolVar(state,
orientation=orientation,
bloxels=orient_bloxels)
# Each piece_oriented_thus is another way a piece can be oriented.
oriented_one_way[piece].constrain(piece_oriented_thus)
for bloxel in orient_bloxels:
occupied_once[bloxel].constrain(piece_oriented_thus)
# Go solve it.
stdout.flush()
stderr.flush()
n_solutions = 0
n_deadends = 0
for is_solution in state.generate_leaves(verbose,
default_guess=default_guess):
if not is_solution:
n_deadends += 1
continue
n_solutions += 1
if not just_count or verbose:
print "==== solution", n_solutions, "depth", "%d," % state.depth(),
print n_deadends, "dead ends ===="
stdout.flush()
if just_count:
continue
# Show a solution.
point_labels = {}
for bloxel in bloxels:
# Those vars overseen by occupied_once[bloxel] that are True.
occupiers = occupied_once[bloxel][True]
assert len(occupiers) == 1
orientation_var = list(occupiers)[0]
piece = orientation_var.orientation.piece
point_labels[bloxel.point] = piece.label
print_points_labels(point_labels)
print
if not multi:
break
return n_solutions, n_deadends
def run_sh(script):
stdout.flush()
stderr.flush()
check_call([str(script)] + sys_argv[1:])
def log_warn(data):
stderr.write(data)
stderr.flush()
def ask_prompt(text):
print(text, end=" ", file=stderr)
stderr.flush()
reply = stdin.readline().rstrip()
print("", file=stderr)
return reply
def eout(emsg):
stderr.write("\n")
stderr.write(" ** Error: %s" % (emsg))
stderr.write("\n")
stderr.flush()
def delayed_(i, fn):
print('\rdispatched: {0:9d} reads'.format(i), end='', file=stderr)
stderr.flush()
return delayed(fn)
