def render_GET(self, request):
log.debug("request from %s: %s" % (self.userid, request.args))
if 'f' in request.args:
fn = request.args['f'][0]
info = self.fileserver.storage.get(fn, False)
if info:
filename, mime, md5sum = info
log.debug("sending file type %s, path %s, md5sum %s" % (mime, filename, md5sum))
genfilename = util.generate_filename(mime)
request.setHeader('content-type', mime)
request.setHeader('content-length', os.path.getsize(filename))
request.setHeader('content-disposition', 'attachment; filename="%s"' % (genfilename))
request.setHeader('x-md5sum', md5sum)
# stream file to the client
fp = open(filename, 'rb')
d = FileSender().beginFileTransfer(fp, request)
def finished(ignored):
fp.close()
request.finish()
d.addErrback(log.error).addCallback(finished)
return server.NOT_DONE_YET
# file not found in extra storage
else:
return self._quick_response(request, 404, 'not found')
return self._quick_response(request, 400, 'bad request')
def post(self):
records = self.get_json_argument('records', None)
with open(generate_filename(settings), 'a') as trace_file:
for record in records:
timestamp = record.pop('timestamp')
trace_file.write("%s: %s\r\n" % (timestamp, json.dumps(record)))
record = massage_record(record, float(timestamp))
RECORD_QUEUE.put(record)
self.set_status(201)
def test_create_records(self):
filename = generate_filename(settings)
assert not os.path.exists(filename)
self._insert_records()
assert os.path.exists(filename)
for line in open(filename):
record = json.loads(line.split(':', 1)[1])
assert record['name'] in ('vehicle_speed',
'fuel_consumed_since_restart',
'longitude', 'latitude')
def LZ_dist_quantifier(quantifiers,
max_model_size,
num_model_orders,
num_chars=4,
num_procs=1,
out_file=None,
order_type='lex'):
""" Generates num_model_orders many different lexicographical- or random-order quantifier representations for each quantifier in quantifiers,
and computes the LZ, bzip2, and gzip complexities for all of those quantifier representations
Args:
quantifiers: a list of quantifier objects
max_model_size: an integer, determining the size (i.e., number of elements) of the quantifier models
num_model_orders: an integer, determing the number of different model order permutations
num_procs: an integer, determining the number of prcessors used for parallel computing
num_chars: an integer, determining the number of characters used in the quantifier representation,
(the number of subareas—--AB, AnotB, BnotA, and neither---of the quantifier model that are considered)
needs to be either 2, 3, or 4
out_file: a string, determining the name of the csv file in which the results are stored
order_type: either 'lex' (for lexicographic model orders) or 'rand' (for random model orders)
Returns:
a pandas data frame which contains the complexity values and other metrics of the quantifiers
"""
time1 = time.time()
data = pd.DataFrame()
pool = Pool(num_procs)
if (order_type == 'lex'):
# get the quantifier representations for the different lexicographical orders for each quantifiers
quantifiers_reps = quantifiers_reps_lex(quantifiers, max_model_size,
num_model_orders, num_chars)
# get uniformities per quantifier
uniformities = [get_uniformity(reps[0]) for reps in quantifiers_reps]
# get quantifier representations for each quantifier per model_order (e.g., quants_reps_per_order = [Q1_rep1, Q2_rep1, ... ])
for order_num in range(num_model_orders):
quants_reps_per_order = [
reps[order_num] for reps in quantifiers_reps
]
# store all the LZ complexities and other metrics in a pandas data frame
data = data.append(pool.map(
get_row_of_table,
zip([max_model_size] * len(quantifiers), quantifiers,
[order_num] * len(quantifiers),
[order_type] * len(quantifiers), quants_reps_per_order,
uniformities, [num_chars] * len(quantifiers))),
ignore_index=True)
if order_type == 'rand':
# make folder for storing the quantifier representations so that they can be retrieved and merged
# when computing the representation for larger sizes (with another function call)
if not os.path.exists("aux"):
os.mkdir("aux")
if not os.path.exists("aux/" + str(num_chars) + "char"):
os.mkdir("aux/" + str(num_chars) + "char")
# get the quantifier representations for one lexicographical orders for each quantifiers
# only include the maximum model size in the representations
min_model_size = max_model_size
if num_chars == 2 and max_model_size == 2:
min_model_size = 1
current_reps = [
q_rep for sublist in quantifiers_reps_lex(
quantifiers, max_model_size, 1, num_chars, min_model_size)
for q_rep in sublist
]
min_model_size = 1
# for the 2 char case model size 1 gives a bug, this is related to a bug in storing a bitarray of size 2 as a binary file
if num_chars == 2:
min_model_size = 2
for order_num in range(num_model_orders):
# make a (new) random-order representation for each quantifier by shuffling
for quant in current_reps:
np.random.shuffle(quant)
# when this function call is for the smallest model size, store the representations so that they can be
# retrieved and merged with a larger model size in a future function call
if max_model_size == min_model_size:
quants_reps = current_reps
for quant_idx in range(len(quantifiers)):
util.store_bitarray(
quants_reps[quant_idx],
"aux/" + util.generate_filename(
quantifiers[quant_idx], max_model_size, num_chars,
order_num))
elif max_model_size > min_model_size:
# merge the quant representations of the current size with the representation of the previous size
# merge them in a random way, in such a way that the order of the elements within
# the current size en previous size representation remain the same
quants_reps = [None for _ in range(len(quantifiers))]
for quant_idx in range(len(quantifiers)):
aux_filename = "aux/" + util.generate_filename(
quantifiers[quant_idx], max_model_size - 1, num_chars,
order_num)
if os.path.exists(aux_filename):
prev_rep = util.load_bitarray(aux_filename)
else:
raise Exception("Needed temporary file not found: " +
aux_filename)
quants_reps[quant_idx] = util.merge_bitarrays_randomly(
prev_rep, current_reps[quant_idx])
# store the merged representation so that they can be retrieved and merged
# with a larger model size in a future function call
util.store_bitarray(
quants_reps[quant_idx],
"aux/" + util.generate_filename(
quantifiers[quant_idx], max_model_size, num_chars,
order_num))
# get uniformities per quantifier
uniformities = [get_uniformity(rep) for rep in quants_reps]
# store all the LZ complexities and other metrics in a pandas data frame
data = data.append(pool.map(
get_row_of_table,
zip([max_model_size] * len(quantifiers), quantifiers,
[order_num] * len(quantifiers),
[order_type] * len(quantifiers), quants_reps, uniformities,
[num_chars] * len(quantifiers))),
ignore_index=True)
# store the data in a csv file
if out_file:
if not os.path.exists("data"):
os.mkdir("data")
if not os.path.exists("data/" + str(num_chars) + "char/"):
os.mkdir("data/" + str(num_chars) + "char/")
if not os.path.exists("data/" + str(num_chars) + "char/" + order_type +
"/"):
os.mkdir("data/" + str(num_chars) + "char/" + order_type + "/")
data.to_csv("data/" + str(num_chars) + "char/" + order_type + "/" +
out_file)
time2 = time.time()
print('max_model_size =', max_model_size, ',', time2 - time1, '\n')
return data
