def render_GET(self, request): log.debug("request from %s: %s" % (self.userid, request.args)) if 'f' in request.args: fn = request.args['f'][0] info = self.fileserver.storage.get(fn, False) if info: filename, mime, md5sum = info log.debug("sending file type %s, path %s, md5sum %s" % (mime, filename, md5sum)) genfilename = util.generate_filename(mime) request.setHeader('content-type', mime) request.setHeader('content-length', os.path.getsize(filename)) request.setHeader('content-disposition', 'attachment; filename="%s"' % (genfilename)) request.setHeader('x-md5sum', md5sum) # stream file to the client fp = open(filename, 'rb') d = FileSender().beginFileTransfer(fp, request) def finished(ignored): fp.close() request.finish() d.addErrback(log.error).addCallback(finished) return server.NOT_DONE_YET # file not found in extra storage else: return self._quick_response(request, 404, 'not found') return self._quick_response(request, 400, 'bad request')
def post(self): records = self.get_json_argument('records', None) with open(generate_filename(settings), 'a') as trace_file: for record in records: timestamp = record.pop('timestamp') trace_file.write("%s: %s\r\n" % (timestamp, json.dumps(record))) record = massage_record(record, float(timestamp)) RECORD_QUEUE.put(record) self.set_status(201)
def test_create_records(self): filename = generate_filename(settings) assert not os.path.exists(filename) self._insert_records() assert os.path.exists(filename) for line in open(filename): record = json.loads(line.split(':', 1)[1]) assert record['name'] in ('vehicle_speed', 'fuel_consumed_since_restart', 'longitude', 'latitude')
def LZ_dist_quantifier(quantifiers, max_model_size, num_model_orders, num_chars=4, num_procs=1, out_file=None, order_type='lex'): """ Generates num_model_orders many different lexicographical- or random-order quantifier representations for each quantifier in quantifiers, and computes the LZ, bzip2, and gzip complexities for all of those quantifier representations Args: quantifiers: a list of quantifier objects max_model_size: an integer, determining the size (i.e., number of elements) of the quantifier models num_model_orders: an integer, determing the number of different model order permutations num_procs: an integer, determining the number of prcessors used for parallel computing num_chars: an integer, determining the number of characters used in the quantifier representation, (the number of subareas—--AB, AnotB, BnotA, and neither---of the quantifier model that are considered) needs to be either 2, 3, or 4 out_file: a string, determining the name of the csv file in which the results are stored order_type: either 'lex' (for lexicographic model orders) or 'rand' (for random model orders) Returns: a pandas data frame which contains the complexity values and other metrics of the quantifiers """ time1 = time.time() data = pd.DataFrame() pool = Pool(num_procs) if (order_type == 'lex'): # get the quantifier representations for the different lexicographical orders for each quantifiers quantifiers_reps = quantifiers_reps_lex(quantifiers, max_model_size, num_model_orders, num_chars) # get uniformities per quantifier uniformities = [get_uniformity(reps[0]) for reps in quantifiers_reps] # get quantifier representations for each quantifier per model_order (e.g., quants_reps_per_order = [Q1_rep1, Q2_rep1, ... ]) for order_num in range(num_model_orders): quants_reps_per_order = [ reps[order_num] for reps in quantifiers_reps ] # store all the LZ complexities and other metrics in a pandas data frame data = data.append(pool.map( get_row_of_table, zip([max_model_size] * len(quantifiers), quantifiers, [order_num] * len(quantifiers), [order_type] * len(quantifiers), quants_reps_per_order, uniformities, [num_chars] * len(quantifiers))), ignore_index=True) if order_type == 'rand': # make folder for storing the quantifier representations so that they can be retrieved and merged # when computing the representation for larger sizes (with another function call) if not os.path.exists("aux"): os.mkdir("aux") if not os.path.exists("aux/" + str(num_chars) + "char"): os.mkdir("aux/" + str(num_chars) + "char") # get the quantifier representations for one lexicographical orders for each quantifiers # only include the maximum model size in the representations min_model_size = max_model_size if num_chars == 2 and max_model_size == 2: min_model_size = 1 current_reps = [ q_rep for sublist in quantifiers_reps_lex( quantifiers, max_model_size, 1, num_chars, min_model_size) for q_rep in sublist ] min_model_size = 1 # for the 2 char case model size 1 gives a bug, this is related to a bug in storing a bitarray of size 2 as a binary file if num_chars == 2: min_model_size = 2 for order_num in range(num_model_orders): # make a (new) random-order representation for each quantifier by shuffling for quant in current_reps: np.random.shuffle(quant) # when this function call is for the smallest model size, store the representations so that they can be # retrieved and merged with a larger model size in a future function call if max_model_size == min_model_size: quants_reps = current_reps for quant_idx in range(len(quantifiers)): util.store_bitarray( quants_reps[quant_idx], "aux/" + util.generate_filename( quantifiers[quant_idx], max_model_size, num_chars, order_num)) elif max_model_size > min_model_size: # merge the quant representations of the current size with the representation of the previous size # merge them in a random way, in such a way that the order of the elements within # the current size en previous size representation remain the same quants_reps = [None for _ in range(len(quantifiers))] for quant_idx in range(len(quantifiers)): aux_filename = "aux/" + util.generate_filename( quantifiers[quant_idx], max_model_size - 1, num_chars, order_num) if os.path.exists(aux_filename): prev_rep = util.load_bitarray(aux_filename) else: raise Exception("Needed temporary file not found: " + aux_filename) quants_reps[quant_idx] = util.merge_bitarrays_randomly( prev_rep, current_reps[quant_idx]) # store the merged representation so that they can be retrieved and merged # with a larger model size in a future function call util.store_bitarray( quants_reps[quant_idx], "aux/" + util.generate_filename( quantifiers[quant_idx], max_model_size, num_chars, order_num)) # get uniformities per quantifier uniformities = [get_uniformity(rep) for rep in quants_reps] # store all the LZ complexities and other metrics in a pandas data frame data = data.append(pool.map( get_row_of_table, zip([max_model_size] * len(quantifiers), quantifiers, [order_num] * len(quantifiers), [order_type] * len(quantifiers), quants_reps, uniformities, [num_chars] * len(quantifiers))), ignore_index=True) # store the data in a csv file if out_file: if not os.path.exists("data"): os.mkdir("data") if not os.path.exists("data/" + str(num_chars) + "char/"): os.mkdir("data/" + str(num_chars) + "char/") if not os.path.exists("data/" + str(num_chars) + "char/" + order_type + "/"): os.mkdir("data/" + str(num_chars) + "char/" + order_type + "/") data.to_csv("data/" + str(num_chars) + "char/" + order_type + "/" + out_file) time2 = time.time() print('max_model_size =', max_model_size, ',', time2 - time1, '\n') return data