def put(self):
"""Receive a sortable reaper or user upload."""
#if not self.uid and not self.drone_request:
# self.abort(402, 'uploads must be from an authorized user or drone')
if 'Content-MD5' not in self.request.headers:
self.abort(400, 'Request must contain a valid "Content-MD5" header.')
filename = self.request.headers.get('Content-Disposition', '').partition('filename=')[2].strip('"')
if not filename:
self.abort(400, 'Request must contain a valid "Content-Disposition" header.')
with tempfile.TemporaryDirectory(prefix='.tmp', dir=self.app.config['upload_path']) as tempdir_path:
filepath = os.path.join(tempdir_path, filename)
success, digest, filesize, duration = util.receive_stream_and_validate(self.request.body_file, filepath, self.request.headers['Content-MD5'])
if not success:
self.abort(400, 'Content-MD5 mismatch.')
if not tarfile.is_tarfile(filepath):
self.abort(415, 'Only tar files are accepted.')
log.info('Received %s [%s] from %s' % (filename, util.hrsize(self.request.content_length), self.request.user_agent))
datainfo = util.parse_file(filepath, digest)
if datainfo is None:
util.quarantine_file(filepath, self.app.config['quarantine_path'])
self.abort(202, 'Quarantining %s (unparsable)' % filename)
util.commit_file(self.app.db.acquisitions, None, datainfo, filepath, self.app.config['data_path'])
util.create_job(self.app.db.acquisitions, datainfo) # FIXME we should only mark files as new and let engine take it from there
throughput = filesize / duration.total_seconds()
log.info('Received %s [%s, %s/s] from %s' % (filename, util.hrsize(filesize), util.hrsize(throughput), self.request.client_addr))
def sort(args):
logging.basicConfig(level=logging.WARNING)
quarantine_path = os.path.join(args.sort_path, 'quarantine')
if not os.path.exists(args.sort_path):
os.makedirs(args.sort_path)
if not os.path.exists(quarantine_path):
os.makedirs(quarantine_path)
print 'initializing DB'
kwargs = dict(tz_aware=True)
db_client = connect_db(args.db_uri, **kwargs)
db = db_client.get_default_database()
print 'inspecting %s' % args.path
files = []
for dirpath, dirnames, filenames in os.walk(args.path):
for filepath in [os.path.join(dirpath, fn) for fn in filenames if not fn.startswith('.')]:
if not os.path.islink(filepath):
files.append(filepath)
dirnames[:] = [dn for dn in dirnames if not dn.startswith('.')] # need to use slice assignment to influence walk behavior
file_cnt = len(files)
print 'found %d files to sort (ignoring symlinks and dotfiles)' % file_cnt
for i, filepath in enumerate(files):
print 'sorting %s [%s] (%d/%d)' % (os.path.basename(filepath), util.hrsize(os.path.getsize(filepath)), i+1, file_cnt)
hash_ = hashlib.sha1()
if not args.quick:
with open(filepath, 'rb') as fd:
for chunk in iter(lambda: fd.read(2**20), ''):
hash_.update(chunk)
datainfo = util.parse_file(filepath, hash_.hexdigest())
if datainfo is None:
util.quarantine_file(filepath, quarantine_path)
print 'Quarantining %s (unparsable)' % os.path.basename(filepath)
else:
util.commit_file(db.acquisitions, None, datainfo, filepath, args.sort_path)
util.create_job(db.acquisitions, datainfo) # FIXME we should only mark files as new and let engine take it from there
def parse_data(metrics):
script_dir = os.path.dirname(os.path.realpath(__file__))
data_dir = script_dir + '/data'
individuals = {}
for data_file in os.listdir(data_dir):
if not data_file in metrics:
logging.info("Skipping %s", data_file)
continue
data_file = data_dir + '/' + data_file
individuals = util.parse_file(data_file, dictionary=individuals)
return individuals
def _parse(input):
"""Compiles file"""
srcfile, state, level = input
LOGGER.debug("Include Parsing {}".format(srcfile))
fpart = os.path.split(srcfile)[1].split('.')[0]
state.symtree.push_scope(fpart, srcfile)
cached = _cached_includes.get(srcfile, None)
if cached:
return state, cached, False
else:
ast = util.parse_file(srcfile, state, level)
_cached_includes[srcfile] = ast
return state, ast, True
def sort(args):
logging.basicConfig(level=logging.WARNING)
quarantine_path = os.path.join(args.sort_path, 'quarantine')
if not os.path.exists(args.sort_path):
os.makedirs(args.sort_path)
if not os.path.exists(quarantine_path):
os.makedirs(quarantine_path)
print 'initializing DB'
kwargs = dict(tz_aware=True)
db_client = connect_db(args.db_uri, **kwargs)
db = db_client.get_default_database()
print 'inspecting %s' % args.path
files = []
for dirpath, dirnames, filenames in os.walk(args.path):
for filepath in [
os.path.join(dirpath, fn) for fn in filenames
if not fn.startswith('.')
]:
if not os.path.islink(filepath):
files.append(filepath)
dirnames[:] = [
dn for dn in dirnames if not dn.startswith('.')
] # need to use slice assignment to influence walk behavior
file_cnt = len(files)
print 'found %d files to sort (ignoring symlinks and dotfiles)' % file_cnt
for i, filepath in enumerate(files):
print 'sorting %s [%s] (%d/%d)' % (os.path.basename(
filepath), util.hrsize(os.path.getsize(filepath)), i + 1, file_cnt)
hash_ = hashlib.sha1()
if not args.quick:
with open(filepath, 'rb') as fd:
for chunk in iter(lambda: fd.read(2**20), ''):
hash_.update(chunk)
datainfo = util.parse_file(filepath, hash_.hexdigest())
if datainfo is None:
util.quarantine_file(filepath, quarantine_path)
print 'Quarantining %s (unparsable)' % os.path.basename(filepath)
else:
util.commit_file(db.acquisitions, None, datainfo, filepath,
args.sort_path)
util.create_job(
db.acquisitions, datainfo
) # FIXME we should only mark files as new and let engine take it from there
def execute(args):
"""Perform verification and actions from cmdline"""
try:
head, absfile = util.validate_file(args.source, 'foidl')
# Construct output file
outhandler = None
if not args.output:
args.output = 'stdout'
outhandler = sys.stdout
else:
outsplit = args.output.split('.')
if len(outsplit) == 1:
args.output = outsplit[0] + _tail_map[args.action]
else:
pass
outhandler = open(args.output, "wt+")
args.inc_paths.append(head)
lvl = ParseLevel.LITE if args.action == 'hdr' else ParseLevel.FULL
hdronly = True if args.action == 'hdr' else False
state = preprocess_runtime(
State(literal_dict(), SymbolTree(absfile),
util.absolutes_path_for(args.inc_paths), hdronly), args)
state.mainsrc = absfile
handler = Handler.handler_for(
args.action,
Bundle(util.absolutes_path_for(args.inc_paths), absfile,
util.parse_file(absfile, state, lvl), state, outhandler,
args.output, args.rt))
handler.validate()
handler.emit()
except (errors.PFoidlError, IOError) as err:
LOGGER.error("{}: {}".format(type(err).__name__, err))
return
sequence_dir = util.get_arg('--sequence_dir', '', parse_func=str)
if util.get_flag("-r") or util.get_flag("--rerun"):
out = run()
log = util.get_flag("-log")
if log:
print('log abs y')
dir = '../tmp'
fn = 'out.zip'
size = os.path.getsize(os.path.join(dir, fn))
print(f'Input file size: {size * 1e-6:0.5f} MB')
if size > 1e8:
print(f'Warning, file too large: {size*1e-6:0.4f} MB')
params, data = util.parse_file(dir, fn, 'out')
# print('uu', data['u'])
N = data['y'][0].shape[0]
ratio = params['y'][0]['aspect_ratio']
Nx, Ny = util.solve_xy_is_a(N, ratio)
Nxy = Nx * Ny
print({'N': N, 'Nx': Nx, 'Ny': Ny, 'eq': Nx * Ny == N})
for k in 'yv':
for i in range(len(data[k])):
data[k][i] = data[k][i][:Nxy]
i = 0
# bins = min(1080, Ny)
bins = Ny
print(f'bw plots ({bins}/1080 ybins)')
def reload(self, log_file):
try:
self.match_info, self.auto_choices, self.log_info, self.lines, self.line_colors = parse_file(
log_file)
except ParseError as e:
messagebox.showerror(
"Error",
"Something went wrong parsing the file: %s. Make sure the file is a valid autonomous log."
% e.message)
return
self.alliance = self.auto_choices["@alliance"]
self.log_name = "%s %s" % (self.match_info["@type"],
self.match_info["@number"])
self.stopwatch.max_time = self.log_info[-1].time
self.stopwatch.stop()
self.step = 0
self.root.title("Tracelog analysis: " + self.log_name)
self.time_slider.configure(to=self.log_info[-1].time)
self.log_window.reset(self.lines, self.line_colors)
self.zmw = None
#print ('Implemention Memory Size:', "{:.2f}".format(impMemKBSize), 'KB (' + str(impMemByteSize), 'Bytes)')
#print ('Cost: $', "{:.2f}".format(0.07*impMemKBSize))
#print ('')
## Output data
#f = open(traceFile, 'r')
#lines = f.read()
#match = re.findall('EIP\s\(([0-9]{2})\):\s([^\s]+)', lines)
#
#for i in range(20):
# print('0x' + match[i][1] + ':' + match[i][0])
#f.close()
#print ('Initialize the timer and counter...Done')
#
#print ('Call the space for Cache Simulator...Done')
tupl = util.parse_file(traceFile)
cache_access_list = tupl[0]
intruction_count = tupl[1]
# Create the cache access list
cache = Cache(cacheSize, blockSize, mapWay, repPolicy)
# Iterate through accesses
get_row_set_from_cache(cache_access_list)
# Print the specified results
print_util.print_formatted_header(traceFile)
print_util.print_generic_header(
cacheSize, blockSize, mapWay,
repPolicy) # ***** Cache Input Parameters *****
print_util.print_calculated_values(cache.get_num_blocks(),
build_response,
parse_file)
from persistencia import Persistencia
if __name__ == '__main__':
input_path = input('Insira caminho para o arquivo de entrada: ')
input_path = Path(input_path).resolve()
print(input_path)
ftype = input('Insira tipo de arquivo (analysis/total): ')
check_file_type(ftype)
delimiter = input('Insira caractere delimitador: ')
check_delimiter_valid(delimiter)
direction = input('Insira orientação desejada (colunas/linhas/c/l): ')
check_direction(direction)
out_path = input('Insira caminho de saida desejado: ')
out_path = Path(out_path).resolve()
filename = ftype + "TimeTab.out"
full_output_path = out_path / filename
print(full_output_path)
content_raw = Persistencia.read_file(input_path)
parsed_content = parse_file(content_raw)
response = build_response(parsed_content, delimiter, direction)
Persistencia.output_file(out_path, filename, response)
if len(sys.argv) < 11:
print('Error: Invalid number of arguments')
print('Usage: [python file] -f [trace file] -s [cache size] -b [block size] -a [associativity] '
'-r [replacement policy]')
sys.exit(3)
# Define the parser arguments
parser = argparse.ArgumentParser()
parser.add_argument('-f', action="store", dest="trace_file", type=util.determine_valid_file)
parser.add_argument('-s', action="store", dest="cache_size", type=util.cache_size_type)
parser.add_argument('-b', action="store", dest="block_size", type=util.block_size_type)
parser.add_argument('-a', action="store", dest="associativity", type=int, choices=[1, 2, 4, 8, 16])
parser.add_argument('-r', action="store", dest="replacement", choices=['RR', 'RND'])
results = parser.parse_args()
# Parse the trace file
cache_access_list = util.parse_file(results.trace_file)
# Create the cache access list
cache = Cache(results.cache_size, results.block_size, results.associativity, results.replacement)
# Iterate through accesses
get_row_set_from_cache(cache_access_list)
# Print the specified results
print_util.print_formatted_header(results.trace_file)
print_util.print_generic_header(results.cache_size, results.block_size, results.associativity, results.replacement)
print_util.print_calculated_values(cache.get_num_blocks(), cache.get_tag_size(), cache.get_indices(),
cache.get_index_size(), cache.get_overhead_size(), cache.get_total_size())
print_util.print_results(cache_accesses, cache_hits, conflict_misses, compulsory_misses)
def solve(input_path): instructions = util.parse_file(input_path) mem = util.process_part2(instructions) result = sum(i[1] for i in mem.items()) return result
def test_parse_file(input_file, expected_out):
base_path = Path(os.getcwd())
f = base_path / "src" / "input_files" / input_file
content = Persistencia.read_file(f)
parsed = parse_file(content)
assert parsed == expected_out
#! config file
import util
config = {
"img":"truck.png",
"maze":{
"width":600,
"height":600,
"item":30,
"space": util.parse_file('ground')
}
}
def train():
"""
モデルの学習
:return: None
"""
# (1)学習データと辞書の取得
questions, answers, word2id, id2word = parse_file(
"../data/conversation_data.txt")
# 文章をidの配列に変換する
ids_questions = sentence_to_word_id(questions, word2id=word2id)
ids_answers = sentence_to_word_id(answers, word2id=word2id)
vocab_size = len(word2id)
# (2)前処理 バケット生成 内部的に形態素解析の幅が長すぎるデータを削除
train_data = create_buckets(ids_questions, ids_answers)
start_time = time.time()
# (3)バケットごとの数の割合を計算
train_bucket_sizes = [len(train_data[b]) for b in range(len(_buckets))]
print(train_bucket_sizes, vocab_size)
# データ数と等しい
train_total_size = float(sum(train_bucket_sizes))
train_buckets_scale = [
sum(train_bucket_sizes[:i + 1]) / train_total_size
for i in range(len(train_bucket_sizes))
]
# tensotflow での処理を開始する。
with tf.Session() as sess:
# (4)モデル作成
model = Seq2SeqModel(vocab_size,
vocab_size,
_buckets,
128,
3,
5.0,
16,
0.5,
0.99,
use_lstm=True)
# 初期化
sess.run(tf.global_variables_initializer())
current_step = 0
step_time, loss = 0.0, 0.0
step_per_checkpoint = 100
step_times = 10000
for step_time_now in range(step_times):
random_number_01 = np.random.random_sample()
# bucket idを選択する。
bucket_id = min([
i for i in range(len(train_buckets_scale))
if train_buckets_scale[i] > random_number_01
])
# (5)バッチ処理を行うデータを選択
encoder_inputs, decoder_inputs, target_weights = model.get_batch(
train_data, bucket_id)
# (6)学習
_, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs,
target_weights, bucket_id, False)
# ステップ時間を計算する。
step_time += (time.time() - start_time) / step_per_checkpoint
current_step += 1
# (7)一定間隔ごとにモデルの評価
if current_step % step_per_checkpoint == 0:
print("step:{} time:{}".format(current_step, step_time))
# 誤差を計算し、描画する。
for bucket_id in range(len(_buckets)):
if len(train_data[bucket_id]) == 0:
print(" eval: empty bucket %d" % (bucket_id))
continue
encoder_inputs, decoder_inputs, target_weights = model.get_batch(
train_data, bucket_id)
_, eval_loss, _ = model.step(sess, encoder_inputs,
decoder_inputs,
target_weights, bucket_id,
True)
print(" eval: bucket %d loss %.2f" %
(bucket_id, eval_loss))
# (8)学習完了時点のモデルをファイル化
model.saver.save(sess, "./tmp/model.ckpt")
# 辞書データを保存する。
json.dump(id2word, open("dictionary_i2w.json", "w"))
json.dump(word2id, open("dictionary_w2i.json", "w"))
def upload(self):
"""
Recieve a multi-file upload.
3 phases:
1 - upload metadata, obtain upload ticket
2 - upload files, one at a time, but in parallel
3 - send a 'complete' message
"""
def store_file(fd, filename, md5, arcpath, arcname):
with tempfile.TemporaryDirectory(prefix='.tmp', dir=self.app.config['upload_path']) as tempdir_path:
filepath = os.path.join(tempdir_path, filename)
success, _, _, _ = util.receive_stream_and_validate(fd, filepath, md5)
if not success:
self.abort(400, 'Content-MD5 mismatch.')
with lockfile.LockFile(arcpath):
with tarfile.open(arcpath, 'a') as archive:
archive.add(filepath, os.path.join(arcname, filename))
if self.public_request:
self.abort(403, 'must be logged in to upload data')
filename = self.request.GET.get('filename')
ticket_id = self.request.GET.get('ticket')
if not ticket_id:
if filename != 'METADATA.json':
self.abort(400, 'first file must be METADATA.json')
try:
json_body = self.request.json_body
jsonschema.validate(json_body, UPLOAD_SCHEMA)
except (ValueError, jsonschema.ValidationError) as e:
self.abort(400, str(e))
filetype = json_body['filetype']
overwrites = json_body['overwrite']
query = {'name': overwrites['project_name'], 'group': overwrites['group_name']}
project = self.app.db.projects.find_one(query) # verify permissions
if not self.superuser_request:
user_perm = util.user_perm(project['permissions'], self.uid)
if not user_perm:
self.abort(403, self.uid + ' does not have permissions on this project')
if users.INTEGER_ROLES[user_perm['access']] < users.INTEGER_ROLES['rw']:
self.abort(403, self.uid + ' does not have at least ' + min_role + ' permissions on this project')
acq_no = overwrites.get('acq_no')
arcname = overwrites['series_uid'] + ('_' + str(acq_no) if acq_no is not None else '') + '_' + filetype
ticket = util.upload_ticket(arcname=arcname) # store arcname for later reference
self.app.db.uploads.insert_one(ticket)
arcpath = os.path.join(self.app.config['upload_path'], ticket['_id'] + '.tar')
store_file(self.request.body_file, filename, self.request.headers['Content-MD5'], arcpath, arcname)
return {'ticket': ticket['_id']}
ticket = self.app.db.uploads.find_one({'_id': ticket_id})
if not ticket:
self.abort(404, 'no such ticket')
arcpath = os.path.join(self.app.config['upload_path'], ticket_id + '.tar')
if self.request.GET.get('complete', '').lower() not in ('1', 'true'):
if 'Content-MD5' not in self.request.headers:
self.app.db.uploads.remove({'_id': ticket_id}) # delete ticket
self.abort(400, 'Request must contain a valid "Content-MD5" header.')
if not filename:
self.app.db.uploads.remove({'_id': ticket_id}) # delete ticket
self.abort(400, 'Request must contain a filename query parameter.')
self.app.db.uploads.update_one({'_id': ticket_id}, {'$set': {'timestamp': datetime.datetime.utcnow()}}) # refresh ticket
store_file(self.request.body_file, filename, self.request.headers['Content-MD5'], arcpath, ticket['arcname'])
else: # complete -> zip, hash, commit
filepath = arcpath[:-2] + 'gz'
with gzip.open(filepath, 'wb', compresslevel=6) as gzfile:
with open(arcpath) as rawfile:
gzfile.writelines(rawfile)
os.remove(arcpath)
sha1 = hashlib.sha1()
with open(filepath, 'rb') as fd:
for chunk in iter(lambda: fd.read(2**20), ''):
sha1.update(chunk)
datainfo = util.parse_file(filepath, sha1.hexdigest())
if datainfo is None:
util.quarantine_file(filepath, self.app.config['quarantine_path'])
self.abort(202, 'Quarantining %s (unparsable)' % filename)
util.commit_file(self.app.db.acquisitions, None, datainfo, filepath, self.app.config['data_path'])
import sys
import argparse
# This is not required if you've installed pycparser into
# your site-packages/ with setup.py
#
sys.path.extend(['.', '..'])
import util
import parser
import gen
if __name__ == "__main__":
# if len(sys.argv) > 1:
# translate_to_c(sys.argv[1])
# else:
# print("Please provide a filename as argument")
ast = util.parse_file(sys.argv[1])
# print(ast)
generator = gen.CodeGenerator(style=gen_s.ExampleCStyle())
print(generator.visit(ast))
from seq2seq_model import Seq2SeqModel
import json
import MeCab
import os
import numpy as np
from util import parse_file, sentence_to_word_id, create_buckets, _buckets, EOS, ignore_list
tagger = MeCab.Tagger("mecabrc")
id2word = json.load(open("dictionary_i2w.json", "r"))
word2id = json.load(open("dictionary_w2i.json", "r"))
_buckets = [(5, 10), (10, 15), (20, 25), (40, 50)]
# 学習データと辞書の取得
questions, answers, _, _ = parse_file("../data/conversation_data.txt")
# 文章をidの配列に変換する
print(questions)
ids_questions = sentence_to_word_id(questions, word2id=word2id)
print(ids_questions)
vocab_size = len(word2id) + 3
print(vocab_size)
ckpt = tf.train.get_checkpoint_state("./tmp")
print(ckpt)
print(tf.train.checkpoint_exists("./tmp/model.ckpt-5000"))
with tf.Session() as sess:
print('init model')
model = Seq2SeqModel(vocab_size,
