def get_db_state(disposition=DISPOSITION_RO): impl = config.get_implementation() if impl is None: impl = sys.modules[__name__] db_filename = config.get_db_filename(impl=impl) lastblock_filename = config.get_lastblock_filename() firstcheck = True for path in [db_filename, lastblock_filename]: if os.path.exists(path): # have already create the db firstcheck = False if not firstcheck and not os.path.exists(lastblock_filename): log.error("FATAL: no such file or directory: %s" % lastblock_filename) # verify that it is well-formed if it exists elif os.path.exists(lastblock_filename): try: with open(lastblock_filename, "r") as f: int(f.read().strip()) except Exception, e: log.error("FATAL: failed to parse: %s " % lastblock_filename) log.exception(e) os.abort()
def get(self, *args, **kwargs): """ GET request """ jsonlist = [] # get the logs (but only the public ones) con = sqlite3.connect(get_db_filename(), detect_types=sqlite3.PARSE_DECLTYPES) cur = con.cursor() # get vehicle name information from vehicle table cur.execute('select UUID, Name from Vehicle') db_tuples = cur.fetchall() vehicle_table = {db_tuple[0]: db_tuple[1] for db_tuple in db_tuples} cur.execute( 'select Id, Date, Description, WindSpeed, Rating, VideoUrl, ErrorLabels, ' 'Source, Feedback, Type from Logs where Public = 1') # need to fetch all here, because we will do more SQL calls while # iterating (having multiple cursor's does not seem to work) db_tuples = cur.fetchall() for db_tuple in db_tuples: jsondict = {} db_data = DBData() log_id = db_tuple[0] jsondict['log_id'] = log_id jsondict['log_date'] = db_tuple[1].strftime('%Y-%m-%d') db_data.description = db_tuple[2] db_data.wind_speed = db_tuple[3] db_data.rating = db_tuple[4] db_data.video_url = db_tuple[5] db_data.error_labels = sorted([int(x) for x in db_tuple[6].split(',') if len(x) > 0]) \ if db_tuple[6] else [] db_data.source = db_tuple[7] db_data.feedback = db_tuple[8] db_data.type = db_tuple[9] jsondict.update(db_data.to_json_dict()) db_data_gen = get_generated_db_data_from_log(log_id, con, cur) if db_data_gen is None: continue jsondict.update(db_data_gen.to_json_dict()) # add vehicle name jsondict['vehicle_name'] = vehicle_table.get( jsondict['vehicle_uuid'], '') airframe_data = get_airframe_data(jsondict['sys_autostart_id']) jsondict['airframe_name'] = airframe_data.get('name', '') \ if airframe_data is not None else '' jsondict['airframe_type'] = airframe_data.get('type', jsondict['sys_autostart_id']) \ if airframe_data is not None else jsondict['sys_autostart_id'] jsonlist.append(jsondict) cur.close() con.close() self.set_header('Content-Type', 'application/json') self.write(json.dumps(jsonlist))
def save( self, block_id, consensus_hash, pending_ops, backup=False ): """ Write out all state to the working directory. Calls the implementation's 'db_save' method. Return True on success Return False on error Raise exception if block_id represents a block we've already processed. """ if block_id < self.lastblock: raise Exception("Already processed up to block %s (got %s)" % (self.lastblock, block_id)) # stage data to temporary files tmp_db_filename = (config.get_db_filename() + ".tmp") tmp_snapshot_filename = (config.get_snapshots_filename() + ".tmp") tmp_lastblock_filename = (config.get_lastblock_filename() + ".tmp") with open(tmp_snapshot_filename, 'w') as f: db_dict = { 'snapshots': self.consensus_hashes } f.write(json.dumps(db_dict)) f.flush() # put this last... with open(tmp_lastblock_filename, "w") as lastblock_f: lastblock_f.write("%s" % block_id) lastblock_f.flush() rc = self.impl.db_save( block_id, consensus_hash, pending_ops, tmp_db_filename, db_state=self.state ) if not rc: # failed to save log.error("Implementation failed to save at block %s to %s" % (block_id, tmp_db_filename)) try: os.unlink( tmp_lastblock_filename ) except: pass try: os.unlink( tmp_snapshot_filename ) except: pass return False rc = self.commit( backup=backup ) if not rc: log.error("Failed to commit data at block %s. Rolling back." % block_id ) self.rollback() return False else: self.lastblock = block_id return True
def commit(self, backup=False, startup=False): """ Move all written but uncommitted data into place. Return True on success Return False on error (in which case the caller should rollback()) It is safe to call this method repeatedly until it returns True. """ if self.read_only: log.error("FATAL: read-only") os.abort() tmp_db_filename = config.get_db_filename(impl=self.impl) + ".tmp" tmp_snapshot_filename = config.get_snapshots_filename(impl=self.impl) + ".tmp" tmp_lastblock_filename = config.get_lastblock_filename(impl=self.impl) + ".tmp" if not os.path.exists(tmp_lastblock_filename) and ( os.path.exists(tmp_db_filename) or os.path.exists(tmp_snapshot_filename)): # we did not successfully stage the write. # rollback log.error("Partial write detected. Not committing.") return False # basic sanity checks: don't overwrite the db if the file is zero bytes, or if we can't load it if os.path.exists(tmp_db_filename): db_dir = os.path.dirname(tmp_db_filename) try: dirfd = os.open(db_dir, os.O_DIRECTORY) os.fsync(dirfd) os.close(dirfd) except Exception, e: log.exception(e) log.error("FATAL: failed to sync directory %s" % db_dir) traceback.print_stack() os.abort() sb = os.stat(tmp_db_filename) if sb.st_size == 0: log.error("Partial write detected: tried to overwrite with zero-sized db! Will rollback.") return False if startup: # make sure we can load this try: with open(tmp_snapshot_filename, "r") as f: db_txt = f.read() db_json = json.loads(db_txt) except: log.error("Partial write detected: corrupt partially-committed db! Will rollback.") return False
def get_backup_paths( cls, block_id, impl, working_dir=None ): """ Get the set of backup paths, given the virtualchain implementation module and block number """ backup_dir = os.path.join( config.get_working_dir(impl=impl, working_dir=working_dir), "backups" ) backup_paths = [] for p in [config.get_db_filename(impl=impl, working_dir=working_dir), config.get_snapshots_filename(impl=impl, working_dir=working_dir), config.get_lastblock_filename(impl=impl, working_dir=working_dir)]: pbase = os.path.basename(p) backup_path = os.path.join( backup_dir, pbase + (".bak.%s" % block_id)) backup_paths.append( backup_path ) return backup_paths
def get_backup_paths( cls, block_id, impl ): """ Get the set of backup paths, given the virtualchain implementation module and block number """ backup_dir = os.path.join( config.get_working_dir(impl=impl), "backups" ) backup_paths = [] for p in [config.get_db_filename(impl=impl), config.get_snapshots_filename(impl=impl), config.get_lastblock_filename(impl=impl)]: pbase = os.path.basename(p) backup_path = os.path.join( backup_dir, pbase + (".bak.%s" % block_id)) backup_paths.append( backup_path ) return backup_paths
def commit( self, backup=False, startup=False ): """ Move all written but uncommitted data into place. Return True on success Return False on error (in which case the caller should rollback()) It is safe to call this method repeatedly until it returns True. """ if self.read_only: log.error("FATAL: read-only") os.abort() tmp_db_filename = config.get_db_filename(impl=self.impl) + ".tmp" tmp_snapshot_filename = config.get_snapshots_filename(impl=self.impl) + ".tmp" tmp_lastblock_filename = config.get_lastblock_filename(impl=self.impl) + ".tmp" if not os.path.exists( tmp_lastblock_filename ) and (os.path.exists(tmp_db_filename) or os.path.exists(tmp_snapshot_filename)): # we did not successfully stage the write. # rollback log.error("Partial write detected. Not committing.") return False # basic sanity checks: don't overwrite the db if the file is zero bytes, or if we can't load it if os.path.exists( tmp_db_filename ): db_dir = os.path.dirname( tmp_db_filename ) try: dirfd = os.open(db_dir, os.O_DIRECTORY) os.fsync(dirfd) os.close( dirfd ) except Exception, e: log.exception(e) log.error("FATAL: failed to sync directory %s" % db_dir) traceback.print_stack() os.abort() sb = os.stat( tmp_db_filename ) if sb.st_size == 0: log.error("Partial write detected: tried to overwrite with zero-sized db! Will rollback.") return False if startup: # make sure we can load this try: with open(tmp_snapshot_filename, "r") as f: db_txt = f.read() db_json = json.loads(db_txt) except: log.error("Partial write detected: corrupt partially-committed db! Will rollback.") return False
def backup_restore( cls, block_id, impl, working_dir=None ): """ Restore from a backup, given the virutalchain implementation module and block number """ backup_dir = os.path.join( config.get_working_dir(impl=impl, working_dir=working_dir), "backups" ) backup_paths = cls.get_backup_paths( block_id, impl, working_dir=working_dir ) for p in backup_paths: assert os.path.exists( p ), "No such backup file: %s" % backup_paths for p in [config.get_db_filename(impl=impl, working_dir=working_dir), config.get_snapshots_filename(impl=impl, working_dir=working_dir), config.get_lastblock_filename(impl=impl, working_dir=working_dir)]: pbase = os.path.basename(p) backup_path = os.path.join( backup_dir, pbase + (".bak.%s" % block_id)) log.debug("Restoring '%s' to '%s'" % (backup_path, p)) shutil.copy( backup_path, p ) return True
def backup_restore( cls, block_id, impl ): """ Restore from a backup, given the virutalchain implementation module and block number """ backup_dir = os.path.join( config.get_working_dir(impl=impl), "backups" ) backup_paths = cls.get_backup_paths( block_id, impl ) for p in backup_paths: assert os.path.exists( p ), "No such backup file: %s" % backup_paths for p in [config.get_db_filename(impl=impl), config.get_snapshots_filename(impl=impl), config.get_lastblock_filename(impl=impl)]: pbase = os.path.basename(p) backup_path = os.path.join( backup_dir, pbase + (".bak.%s" % block_id)) log.debug("Restoring '%s' to '%s'" % (backup_path, p)) shutil.copy( backup_path, p ) return True
def save(self, block_id, consensus_hash, pending_ops, backup=False): """ Write out all state to the working directory. Calls the implementation's 'db_save' method to store any state for this block. Calls the implementation's 'db_continue' method at the very end, to signal to the implementation that all virtualchain state has been saved. This method can return False, in which case, indexing stops Return True on success Return False if the implementation wants to exit. Aborts on fatal error """ if self.read_only: log.error("FATAL: read only") traceback.print_stack() os.abort() if block_id < self.lastblock: log.error("FATAL: Already processed up to block %s (got %s)" % (self.lastblock, block_id)) traceback.print_stack() os.abort() # stage data to temporary files tmp_db_filename = (config.get_db_filename(impl=self.impl) + ".tmp") tmp_snapshot_filename = (config.get_snapshots_filename(impl=self.impl) + ".tmp") tmp_lastblock_filename = (config.get_lastblock_filename(impl=self.impl) + ".tmp") try: with open(tmp_snapshot_filename, 'w') as f: db_dict = { 'snapshots': self.consensus_hashes } f.write(json.dumps(db_dict)) f.flush() with open(tmp_lastblock_filename, "w") as lastblock_f: lastblock_f.write("%s" % block_id) lastblock_f.flush() except Exception, e: # failure to save is fatal log.exception(e) log.error("FATAL: Could not stage data for block %s" % block_id) traceback.print_stack() os.abort()
def rollback( self ): """ Roll back a pending write: blow away temporary files. """ tmp_db_filename = config.get_db_filename() + ".tmp" tmp_snapshot_filename = config.get_snapshots_filename() + ".tmp" tmp_lastblock_filename = config.get_lastblock_filename() + ".tmp" for f in [tmp_db_filename, tmp_snapshot_filename, tmp_lastblock_filename]: if os.path.exists( f ): try: os.unlink( f ) except: log.error("Failed to unlink '%s'" % f ) pass
def save( self, block_id, consensus_hash, pending_ops, backup=False ): """ Write out all state to the working directory. Calls the implementation's 'db_save' method to store any state for this block. Calls the implementation's 'db_continue' method at the very end, to signal to the implementation that all virtualchain state has been saved. This method can return False, in which case, indexing stops Return True on success Return False if the implementation wants to exit. Aborts on fatal error """ if self.read_only: log.error("FATAL: read only") traceback.print_stack() os.abort() if block_id < self.lastblock: log.error("FATAL: Already processed up to block %s (got %s)" % (self.lastblock, block_id)) traceback.print_stack() os.abort() # stage data to temporary files tmp_db_filename = (config.get_db_filename(impl=self.impl) + ".tmp") tmp_snapshot_filename = (config.get_snapshots_filename(impl=self.impl) + ".tmp") tmp_lastblock_filename = (config.get_lastblock_filename(impl=self.impl) + ".tmp") try: with open(tmp_snapshot_filename, 'w') as f: db_dict = { 'snapshots': self.consensus_hashes } f.write(json.dumps(db_dict)) f.flush() with open(tmp_lastblock_filename, "w") as lastblock_f: lastblock_f.write("%s" % block_id) lastblock_f.flush() except Exception, e: # failure to save is fatal log.exception(e) log.error("FATAL: Could not stage data for block %s" % block_id) traceback.print_stack() os.abort()
def generate_db_data_from_log_file(log_id, db_connection=None): """ Extract necessary information from the log file and insert as an entry to the LogsGenerated table (faster information retrieval later on). This is an expensive operation. It's ok to call this a second time for the same log, the call will just silently fail (but still read the whole log and will not update the DB entry) :return: DBDataGenerated object """ db_data_gen = DBDataGenerated.from_log_file(log_id) need_closing = False if db_connection is None: db_connection = sqlite3.connect(get_db_filename()) need_closing = True db_cursor = db_connection.cursor() try: db_cursor.execute( 'insert into LogsGenerated (Id, Duration, ' 'Mavtype, Estimator, AutostartId, Hardware, ' 'Software, NumLoggedErrors, NumLoggedWarnings, ' 'FlightModes, SoftwareVersion, UUID, FlightModeDurations, StartTime) values ' '(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [ log_id, db_data_gen.duration_s, db_data_gen.mav_type, db_data_gen.estimator, db_data_gen.sys_autostart_id, db_data_gen.sys_hw, db_data_gen.ver_sw, db_data_gen.num_logged_errors, db_data_gen.num_logged_warnings, ','.join(map(str, db_data_gen.flight_modes)), db_data_gen.ver_sw_release, db_data_gen.vehicle_uuid, db_data_gen.flight_mode_durations_str(), db_data_gen.start_time_utc ]) db_connection.commit() except sqlite3.IntegrityError: # someone else already inserted it (race). just ignore it pass db_cursor.close() if need_closing: db_connection.close() return db_data_gen
def delete_log_entry(log_id, token): """ delete a log entry (DB & file), validate token first :return: True on success """ con = sqlite3.connect(get_db_filename(), detect_types=sqlite3.PARSE_DECLTYPES) cur = con.cursor() cur.execute('select Token from Logs where Id = ?', (log_id, )) db_tuple = cur.fetchone() if db_tuple is None: return False if token != db_tuple[0]: # validate token return False # kml file kml_path = get_kml_filepath() kml_file_name = os.path.join(kml_path, log_id.replace('/', '.') + '.kml') if os.path.exists(kml_file_name): os.unlink(kml_file_name) #preview image preview_image_filename = os.path.join(get_overview_img_filepath(), log_id + '.png') if os.path.exists(preview_image_filename): os.unlink(preview_image_filename) log_file_name = get_log_filename(log_id) print('deleting log entry {} and file {}'.format( log_id, log_file_name)) os.unlink(log_file_name) cur.execute("DELETE FROM LogsGenerated WHERE Id = ?", (log_id, )) cur.execute("DELETE FROM Logs WHERE Id = ?", (log_id, )) con.commit() cur.close() con.close() # need to clear the cache as well clear_ulog_cache() return True
def get_original_filename(default_value, new_file_suffix): """ get the uploaded file name & exchange the file extension """ try: con = sqlite3.connect(get_db_filename(), detect_types=sqlite3.PARSE_DECLTYPES) cur = con.cursor() cur.execute('select OriginalFilename ' 'from Logs where Id = ?', [log_id]) db_tuple = cur.fetchone() if db_tuple is not None: original_file_name = escape(db_tuple[0]) if original_file_name[-4:].lower() == '.ulg': original_file_name = original_file_name[:-4] return original_file_name + new_file_suffix cur.close() con.close() except: print("DB access failed:", sys.exc_info()[0], sys.exc_info()[1]) return default_value
def get(self, *args, **kwargs): jsonlist = list() # get the logs (but only the public ones) con = sqlite3.connect(get_db_filename(), detect_types=sqlite3.PARSE_DECLTYPES) cur = con.cursor() cur.execute('select Id, Date, Description, WindSpeed, Rating, VideoUrl, ErrorLabels ' 'from Logs where Public = 1') # need to fetch all here, because we will do more SQL calls while # iterating (having multiple cursor's does not seem to work) db_tuples = cur.fetchall() for db_tuple in db_tuples: jsondict = dict() db_data = DBData() log_id = db_tuple[0] jsondict['log_id'] = log_id jsondict['log_date'] = db_tuple[1].strftime('%Y-%m-%d') db_data.description = db_tuple[2] db_data.feedback = '' db_data.type = '' db_data.wind_speed = db_tuple[3] db_data.rating = db_tuple[4] db_data.video_url = db_tuple[5] db_data.error_labels = sorted([int(x) for x in db_tuple[6].split(',') if len(x) > 0]) \ if db_tuple[6] else [] jsondict.update(db_data.to_json_dict()) db_data_gen = get_generated_db_data_from_log(log_id, con, cur) if db_data_gen is None: continue jsondict.update(db_data_gen.to_json_dict()) jsonlist.append(jsondict) cur.close() con.close() self.set_header('Content-Type', 'application/json') self.write(json.dumps(jsonlist))
def make_backups( self, block_id ): """ If we're doing backups on a regular basis, then carry them out here if it is time to do so. This method does nothing otherwise. Abort on failure """ # make a backup? if self.backup_frequency is not None: if (block_id % self.backup_frequency) == 0: backup_dir = os.path.join( config.get_working_dir(impl=self.impl), "backups" ) if not os.path.exists(backup_dir): try: os.makedirs(backup_dir) except Exception, e: log.exception(e) log.error("FATAL: failed to make backup directory '%s'" % backup_dir) traceback.print_stack() os.abort() for p in [config.get_db_filename(impl=self.impl), config.get_snapshots_filename(impl=self.impl), config.get_lastblock_filename(impl=self.impl)]: if os.path.exists(p): try: pbase = os.path.basename(p) backup_path = os.path.join( backup_dir, pbase + (".bak.%s" % (block_id - 1))) if not os.path.exists( backup_path ): shutil.copy( p, backup_path ) else: log.error("Will not overwrite '%s'" % backup_path) except Exception, e: log.exception(e) log.error("FATAL: failed to back up '%s'" % p) traceback.print_stack() os.abort()
def make_backups( self, block_id, working_dir=None ): """ If we're doing backups on a regular basis, then carry them out here if it is time to do so. This method does nothing otherwise. Abort on failure """ # make a backup? if self.backup_frequency is not None: if (block_id % self.backup_frequency) == 0: backup_dir = os.path.join( config.get_working_dir(impl=self.impl, working_dir=working_dir), "backups" ) if not os.path.exists(backup_dir): try: os.makedirs(backup_dir) except Exception, e: log.exception(e) log.error("FATAL: failed to make backup directory '%s'" % backup_dir) traceback.print_stack() os.abort() for p in [config.get_db_filename(impl=self.impl, working_dir=working_dir), config.get_snapshots_filename(impl=self.impl, working_dir=working_dir), config.get_lastblock_filename(impl=self.impl, working_dir=working_dir)]: if os.path.exists(p): try: pbase = os.path.basename(p) backup_path = os.path.join( backup_dir, pbase + (".bak.%s" % (block_id - 1))) if not os.path.exists( backup_path ): shutil.copy( p, backup_path ) else: log.error("Will not overwrite '%s'" % backup_path) except Exception, e: log.exception(e) log.error("FATAL: failed to back up '%s'" % p) traceback.print_stack() os.abort()
def post(self, *args, **kwargs): """ POST request callback """ if self.multipart_streamer: try: self.multipart_streamer.data_complete() form_data = self.multipart_streamer.get_values([ 'description', 'email', 'allowForAnalysis', 'obfuscated', 'source', 'type', 'feedback', 'windSpeed', 'rating', 'videoUrl', 'public', 'vehicleName' ]) description = escape(form_data['description'].decode("utf-8")) email = form_data.get('email', bytes("(no email provided)", 'utf-8')).decode("utf-8") upload_type = form_data.get('type', bytes("personal", 'utf-8')).decode("utf-8") source = form_data.get('source', bytes("webui", 'utf-8')).decode("utf-8") title = '' # may be used in future... obfuscated = { 'true': 1, 'false': 0 }.get( form_data.get('obfuscated', b'false').decode('utf-8'), 0) allow_for_analysis = { 'true': 1, 'false': 0 }.get( form_data.get('allowForAnalysis', b'false').decode('utf-8'), 0) feedback = escape( form_data.get('feedback', b'').decode("utf-8")) wind_speed = -1 rating = '' video_url = '' is_public = 1 vehicle_name = escape( form_data.get('vehicleName', bytes("", 'utf-8')).decode("utf-8")) error_labels = '' # TODO: make the format of formdict a little more compatible with form_data above formdict = {} formdict['description'] = description formdict['email'] = email formdict['upload_type'] = upload_type formdict['source'] = source formdict['title'] = title formdict['obfuscated'] = obfuscated formdict['allow_for_analysis'] = allow_for_analysis formdict['feedback'] = feedback formdict['wind_speed'] = wind_speed formdict['rating'] = rating formdict['video_url'] = video_url formdict['is_public'] = is_public formdict['vehicle_name'] = vehicle_name formdict['error_labels'] = error_labels # we don't bother parsing any of the "flight report" metadata, it's not very useful to us # stored_email = '' # if upload_type == 'flightreport': # try: # wind_speed = int(escape(form_data['windSpeed'].decode("utf-8"))) # except ValueError: # wind_speed = -1 # rating = escape(form_data['rating'].decode("utf-8")) # if rating == 'notset': rating = '' # stored_email = email # # get video url & check if valid # video_url = escape(form_data['videoUrl'].decode("utf-8"), quote=True) # if not validate_url(video_url): # video_url = '' # if 'vehicleName' in form_data: # vehicle_name = escape(form_data['vehicleName'].decode("utf-8")) # # always allow for statistical analysis # allow_for_analysis = 1 # if 'public' in form_data: # if form_data['public'].decode("utf-8") == 'true': # is_public = 1 # open the database connection con = sqlite3.connect(get_db_filename()) cur = con.cursor() file_obj = self.multipart_streamer.get_parts_by_name( 'filearg')[0] upload_file_name = file_obj.get_filename() # read file header and ensure validity peek_ulog_header = file_obj.get_payload_partial( len(ULog.HEADER_BYTES)) peek_zip_header = file_obj.get_payload_partial(4) zip_headers = [ b'\x50\x4b\x03\x04', b'\x50\x4b\x05\x06', b'\x50\x4b\x07\x08' ] # we check that it is either a well formed zip or ULog # is file a ULog? then continue as we were :) if (peek_ulog_header == ULog.HEADER_BYTES): log_id = save_uploaded_log(con, cur, file_obj, formdict) # generate URL info and redirect url = '/plot_app?log=' + log_id full_plot_url = get_http_protocol( ) + '://' + get_domain_name() + url print(full_plot_url) # do not redirect for QGC if source != 'QGroundControl': self.redirect(url) # is the file a zip? read the magic numbers and unzip it elif (peek_zip_header in zip_headers): with zipfile.ZipFile(file_obj.f_out) as zip: for log_filename in zip.namelist(): # make sure we're dealing with a ulog file # TODO: do actual validation here, don't just check filename _, ext = os.path.splitext(log_filename) if ext not in ['.ulg', '.ulog']: print( f'Skipping extracting non-ULog file {file_obj.f_out.name}//{log_filename}' ) continue # TODO: switch to save_uploaded_log # generate a log ID and persistence filename while True: log_id = str(uuid.uuid4()) new_file_name = get_log_filename(log_id) if not os.path.exists(new_file_name): break # extract and rename the ulog file to something we control print( f'Extracting uploaded log {file_obj.f_out.name}//{log_filename} file to', new_file_name) zip.extract(log_filename, path=os.path.dirname(new_file_name)) os.rename( os.path.join(os.path.dirname(new_file_name), log_filename), new_file_name) # Load the ulog file but only if not uploaded via CI. ulog = None if source != 'CI': ulog_file_name = get_log_filename(log_id) ulog = load_ulog_file(ulog_file_name) # generate a token: secure random string (url-safe) token = str(binascii.hexlify(os.urandom(16)), 'ascii') # put additional data into a DB cur.execute( 'insert into Logs (Id, Title, Description, ' 'OriginalFilename, Date, AllowForAnalysis, Obfuscated, ' 'Source, Email, WindSpeed, Rating, Feedback, Type, ' 'videoUrl, ErrorLabels, Public, Token) values ' '(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [ log_id, title, description, upload_file_name, datetime.datetime.now(), allow_for_analysis, obfuscated, source, email, wind_speed, rating, feedback, upload_type, video_url, error_labels, is_public, token ]) if ulog is not None: vehicle_data = update_vehicle_db_entry( cur, ulog, log_id, vehicle_name) vehicle_name = vehicle_data.name con.commit() generate_db_data_from_log_file(log_id, con) con.commit() # generate URL info and redirect url = '/plot_app?log=' + log_id full_plot_url = get_http_protocol( ) + '://' + get_domain_name() + url print(full_plot_url) self.redirect('/browse') # is file neither a zip nor a ULog? error out :) else: if upload_file_name[-7:].lower() == '.px4log': raise CustomHTTPError( 400, 'Invalid File. This seems to be a px4log file. ' 'Upload it to <a href="http://logs.uaventure.com" ' 'target="_blank">logs.uaventure.com</a>.') raise CustomHTTPError(400, 'Invalid File') # this massive chunk of comment was the code used to send emails for # uploaded flight reports. we no longer use this functionality. # however, for some weird reason, this chunk of code also generated a # LogsGenerated entry for faster log loading for public logs. so # we move the line up and out of the code it's not supposed to be a part # of, and put it right here :) #generate_db_data_from_log_file(log_id, con) # delete_url = get_http_protocol()+'://'+get_domain_name()+ \ # '/edit_entry?action=delete&log='+log_id+'&token='+token # information for the notification email # info = {} # info['description'] = description # info['feedback'] = feedback # info['upload_filename'] = upload_file_name # info['type'] = '' # info['airframe'] = '' # info['hardware'] = '' # info['uuid'] = '' # info['software'] = '' # info['rating'] = rating # if len(vehicle_name) > 0: # info['vehicle_name'] = vehicle_name # if ulog is not None: # px4_ulog = PX4ULog(ulog) # info['type'] = px4_ulog.get_mav_type() # airframe_name_tuple = get_airframe_name(ulog) # if airframe_name_tuple is not None: # airframe_name, airframe_id = airframe_name_tuple # if len(airframe_name) == 0: # info['airframe'] = airframe_id # else: # info['airframe'] = airframe_name # sys_hardware = '' # if 'ver_hw' in ulog.msg_info_dict: # sys_hardware = escape(ulog.msg_info_dict['ver_hw']) # info['hardware'] = sys_hardware # if 'sys_uuid' in ulog.msg_info_dict and sys_hardware != 'SITL': # info['uuid'] = escape(ulog.msg_info_dict['sys_uuid']) # branch_info = '' # if 'ver_sw_branch' in ulog.msg_info_dict: # branch_info = ' (branch: '+ulog.msg_info_dict['ver_sw_branch']+')' # if 'ver_sw' in ulog.msg_info_dict: # ver_sw = escape(ulog.msg_info_dict['ver_sw']) # info['software'] = ver_sw + branch_info # if upload_type == 'flightreport' and is_public and source != 'CI': # destinations = set(email_notifications_config['public_flightreport']) # if rating in ['unsatisfactory', 'crash_sw_hw', 'crash_pilot']: # destinations = destinations | \ # set(email_notifications_config['public_flightreport_bad']) # send_flightreport_email( # list(destinations), # full_plot_url, # DBData.rating_str_static(rating), # DBData.wind_speed_str_static(wind_speed), delete_url, # stored_email, info) # # also generate the additional DB entry # # (we may have the log already loaded in 'ulog', however the # # lru cache will make it very quick to load it again) # generate_db_data_from_log_file(log_id, con) # # also generate the preview image # IOLoop.instance().add_callback(generate_overview_img_from_id, log_id) # send notification emails # send_notification_email(email, full_plot_url, delete_url, info) except CustomHTTPError: raise except ULogException as e: raise CustomHTTPError( 400, 'Failed to parse the file. It is most likely corrupt.' ) from e except Exception as e: print('Fatal error when handling POST data', sys.exc_info()[0], sys.exc_info()[1]) traceback.print_exc() raise CustomHTTPError(500) from e finally: # close our DB connections cur.close() con.close() # free the uploaded files self.multipart_streamer.release_parts()
def __init__(self, plot_config, verbose_output=False): self._config = plot_config self._verbose_output = verbose_output # lists of dates when a _log was uploaded, one list per type self._public_logs_dates = [] self._private_logs_dates = [] self._ci_logs_dates = [] self._all_logs_dates = [] self._public_logs = [] # read from the DB con = sqlite3.connect(get_db_filename(), detect_types=sqlite3.PARSE_DECLTYPES) with con: cur = con.cursor() cur.execute('select Id, Date, Source, Public, Rating from Logs') db_tuples = cur.fetchall() for db_tuple in db_tuples: log = _Log(db_tuple) self._all_logs_dates.append(log.date) if log.is_public == 1: if log.source == 'CI': self._ci_logs_dates.append(log.date) else: self._public_logs_dates.append(log.date) else: if log.source == 'CI': self._ci_logs_dates.append(log.date) else: self._private_logs_dates.append(log.date) # LogsGenerated: public only if log.is_public != 1 or log.source == 'CI': continue cur.execute('select * from LogsGenerated where Id = ?', [log.log_id]) db_tuple = cur.fetchone() if db_tuple is None: print("Error: no generated data") continue log.set_generated(db_tuple) # filter bogus entries if log.sw_version == 'v0.0.0': if self._verbose_output: print('Warning: %s with version=v0.0.0' % log.log_id) continue if log.duration > 7 * 24 * 3600: # probably bogus timestamp(s) if self._verbose_output: print('Warning: %s with very high duration %i' % (log.log_id, log.duration)) continue if log.sw_version == '': # FIXME: does that still occur and if so why? if self._verbose_output: print('Warning: %s version not set' % log.log_id) continue if log.autostart_id == 0: print('Warning: %s with autostart_id=0' % log.log_id) continue try: ver_major = int(log.sw_version[1:].split('.')[0]) if ver_major >= 2 or ver_major == 0: print('Warning: %s with large/small version %s' % (log.log_id, log.sw_version)) continue except: continue self._public_logs.append(log) self._version_data = {} # dict of _VersionData items self._all_airframes = set() self._all_boards = set() self._all_ratings = set() self._all_flight_modes = set() self._total_duration = 0 # in hours, public logs only self._total_last_version_duration = 0 # in hours, public logs only self._latest_major_release = "" for log in self._public_logs: if not log.sw_version in self._version_data: self._version_data[log.sw_version] = _VersionData() self._all_airframes.add(str(log.autostart_id)) self._all_boards.add(log.hardware) self._all_ratings.add(log.rating) cur_version_data = self._version_data[log.sw_version] boards = cur_version_data.boards boards_num_logs = cur_version_data.boards_num_logs airframes = cur_version_data.airframes airframes_num_logs = cur_version_data.airframes_num_logs ratings = cur_version_data.ratings flight_modes = cur_version_data.flight_mode_durations if not log.hardware in boards: boards[log.hardware] = 0 boards_num_logs[log.hardware] = 0 boards[log.hardware] += log.duration / 3600. boards_num_logs[log.hardware] += 1 for flight_mode, duration in log.flight_mode_durations: flight_mode_str = str(flight_mode) self._all_flight_modes.add(flight_mode_str) if not flight_mode_str in flight_modes: flight_modes[flight_mode_str] = 0. flight_modes[flight_mode_str] += duration / 3600. autostart_str = str(log.autostart_id) if not autostart_str in airframes: airframes[autostart_str] = 0 airframes_num_logs[autostart_str] = 0 airframes[autostart_str] += log.duration / 3600. airframes_num_logs[autostart_str] += 1 if not log.rating in ratings: ratings[log.rating] = 0 ratings[log.rating] += 1 self._total_duration += log.duration / 3600. if len(self._version_data) > 0: latest_version = sorted(self._version_data, key=functools.cmp_to_key( _Log.compare_version))[-1] latest_major_version = latest_version.split('.')[0:2] self._latest_major_release = '.'.join(latest_major_version) for log in self._public_logs: if log.sw_version.split('.')[0:2] == latest_major_version: self._total_last_version_duration += log.duration / 3600.
def save(self, block_id, consensus_hash, pending_ops, backup=False): """ Write out all state to the working directory. Calls the implementation's 'db_save' method. Return True on success Return False on error Raise exception if block_id represents a block we've already processed. """ if block_id < self.lastblock: raise Exception("Already processed up to block %s (got %s)" % (self.lastblock, block_id)) # stage data to temporary files tmp_db_filename = (config.get_db_filename() + ".tmp") tmp_snapshot_filename = (config.get_snapshots_filename() + ".tmp") tmp_lastblock_filename = (config.get_lastblock_filename() + ".tmp") with open(tmp_snapshot_filename, 'w') as f: db_dict = {'snapshots': self.consensus_hashes} f.write(json.dumps(db_dict)) f.flush() # put this last... with open(tmp_lastblock_filename, "w") as lastblock_f: lastblock_f.write("%s" % block_id) lastblock_f.flush() rc = self.impl.db_save(block_id, consensus_hash, pending_ops, tmp_db_filename, db_state=self.state) if not rc: # failed to save log.error("Implementation failed to save at block %s to %s" % (block_id, tmp_db_filename)) try: os.unlink(tmp_lastblock_filename) except: pass try: os.unlink(tmp_snapshot_filename) except: pass return False rc = self.commit(backup=backup) if not rc: log.error("Failed to commit data at block %s. Rolling back." % block_id) self.rollback() return False else: self.lastblock = block_id return True
def commit(self, backup=False, startup=False): """ Move all written but uncommitted data into place. Return True on success Return False on error (in which case the caller should rollback()) It is safe to call this method repeatedly until it returns True. """ tmp_db_filename = config.get_db_filename() + ".tmp" tmp_snapshot_filename = config.get_snapshots_filename() + ".tmp" tmp_lastblock_filename = config.get_lastblock_filename() + ".tmp" if not os.path.exists(tmp_lastblock_filename) and ( os.path.exists(tmp_db_filename) or os.path.exists(tmp_snapshot_filename)): # we did not successfully stage the write. # rollback log.error("Partial write detected. Not committing.") return False # basic sanity checks: don't overwrite the db if the file is zero bytes, or if we can't load it if os.path.exists(tmp_db_filename): sb = os.stat(tmp_db_filename) if sb.st_size == 0: log.error( "Partial write detected: tried to overwrite with zero-sized db! Will rollback." ) return False if startup: # make sure we can load this try: with open(tmp_snapshot_filename, "r") as f: db_txt = f.read() db_json = json.loads(db_txt) except: log.error( "Partial write detected: corrupt partially-committed db! Will rollback." ) return False backup_time = int(time.time() * 1000000) for tmp_filename, filename in zip( [tmp_lastblock_filename, tmp_snapshot_filename, tmp_db_filename], \ [config.get_lastblock_filename(), config.get_snapshots_filename(), config.get_db_filename()] ): if not os.path.exists(tmp_filename): continue # commit our new lastblock, consensus hash set, and state engine data try: # NOTE: rename fails on Windows if the destination exists if sys.platform == 'win32' and os.path.exists(filename): try: os.unlink(filename) except: pass if not backup: os.rename(tmp_filename, filename) else: shutil.copy(tmp_filename, filename) os.rename(tmp_filename, tmp_filename + (".%s" % backup_time)) except Exception, e: log.exception(e) return False
def post(self, *args, **kwargs): """ POST request callback """ if self.multipart_streamer: try: self.multipart_streamer.data_complete() form_data = self.multipart_streamer.get_values( ['description', 'email', 'allowForAnalysis', 'obfuscated', 'source', 'type', 'feedback', 'windSpeed', 'rating', 'videoUrl', 'public', 'vehicleName']) description = escape(form_data['description'].decode("utf-8")) email = form_data['email'].decode("utf-8") upload_type = 'personal' if 'type' in form_data: upload_type = form_data['type'].decode("utf-8") source = 'webui' title = '' # may be used in future... if 'source' in form_data: source = form_data['source'].decode("utf-8") obfuscated = 0 if 'obfuscated' in form_data: if form_data['obfuscated'].decode("utf-8") == 'true': obfuscated = 1 allow_for_analysis = 0 if 'allowForAnalysis' in form_data: if form_data['allowForAnalysis'].decode("utf-8") == 'true': allow_for_analysis = 1 feedback = '' if 'feedback' in form_data: feedback = escape(form_data['feedback'].decode("utf-8")) wind_speed = -1 rating = '' stored_email = '' video_url = '' is_public = 0 vehicle_name = '' error_labels = '' if upload_type == 'flightreport': try: wind_speed = int(escape(form_data['windSpeed'].decode("utf-8"))) except ValueError: wind_speed = -1 rating = escape(form_data['rating'].decode("utf-8")) if rating == 'notset': rating = '' stored_email = email # get video url & check if valid video_url = escape(form_data['videoUrl'].decode("utf-8"), quote=True) if not validate_url(video_url): video_url = '' if 'vehicleName' in form_data: vehicle_name = escape(form_data['vehicleName'].decode("utf-8")) # always allow for statistical analysis allow_for_analysis = 1 if 'public' in form_data: if form_data['public'].decode("utf-8") == 'true': is_public = 1 file_obj = self.multipart_streamer.get_parts_by_name('filearg')[0] upload_file_name = file_obj.get_filename() while True: log_id = str(uuid.uuid4()) new_file_name = get_log_filename(log_id) if not os.path.exists(new_file_name): break # read file header & check if really an ULog file header_len = len(ULog.HEADER_BYTES) if (file_obj.get_payload_partial(header_len) != ULog.HEADER_BYTES): if upload_file_name[-7:].lower() == '.px4log': raise CustomHTTPError( 400, 'Invalid File. This seems to be a px4log file. ' 'Upload it to <a href="http://logs.uaventure.com" ' 'target="_blank">logs.uaventure.com</a>.') raise CustomHTTPError(400, 'Invalid File') print('Moving uploaded file to', new_file_name) file_obj.move(new_file_name) if obfuscated == 1: # TODO: randomize gps data, ... pass # generate a token: secure random string (url-safe) token = str(binascii.hexlify(os.urandom(16)), 'ascii') # Load the ulog file but only if not uploaded via CI. # Then we open the DB connection. ulog = None if source != 'CI': ulog_file_name = get_log_filename(log_id) ulog = load_ulog_file(ulog_file_name) # put additional data into a DB con = sqlite3.connect(get_db_filename()) cur = con.cursor() cur.execute( 'insert into Logs (Id, Title, Description, ' 'OriginalFilename, Date, AllowForAnalysis, Obfuscated, ' 'Source, Email, WindSpeed, Rating, Feedback, Type, ' 'videoUrl, ErrorLabels, Public, Token) values ' '(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [log_id, title, description, upload_file_name, datetime.datetime.now(), allow_for_analysis, obfuscated, source, stored_email, wind_speed, rating, feedback, upload_type, video_url, error_labels, is_public, token]) if ulog is not None: vehicle_data = update_vehicle_db_entry(cur, ulog, log_id, vehicle_name) vehicle_name = vehicle_data.name con.commit() url = '/plot_app?log='+log_id full_plot_url = get_http_protocol()+'://'+get_domain_name()+url print(full_plot_url) delete_url = get_http_protocol()+'://'+get_domain_name()+ \ '/edit_entry?action=delete&log='+log_id+'&token='+token # information for the notification email info = {} info['description'] = description info['feedback'] = feedback info['upload_filename'] = upload_file_name info['type'] = '' info['airframe'] = '' info['hardware'] = '' info['uuid'] = '' info['software'] = '' info['rating'] = rating if len(vehicle_name) > 0: info['vehicle_name'] = vehicle_name if ulog is not None: px4_ulog = PX4ULog(ulog) info['type'] = px4_ulog.get_mav_type() airframe_name_tuple = get_airframe_name(ulog) if airframe_name_tuple is not None: airframe_name, airframe_id = airframe_name_tuple if len(airframe_name) == 0: info['airframe'] = airframe_id else: info['airframe'] = airframe_name sys_hardware = '' if 'ver_hw' in ulog.msg_info_dict: sys_hardware = escape(ulog.msg_info_dict['ver_hw']) info['hardware'] = sys_hardware if 'sys_uuid' in ulog.msg_info_dict and sys_hardware != 'SITL': info['uuid'] = escape(ulog.msg_info_dict['sys_uuid']) branch_info = '' if 'ver_sw_branch' in ulog.msg_info_dict: branch_info = ' (branch: '+ulog.msg_info_dict['ver_sw_branch']+')' if 'ver_sw' in ulog.msg_info_dict: ver_sw = escape(ulog.msg_info_dict['ver_sw']) info['software'] = ver_sw + branch_info if upload_type == 'flightreport' and is_public: destinations = set(email_notifications_config['public_flightreport']) if rating in ['unsatisfactory', 'crash_sw_hw', 'crash_pilot']: destinations = destinations | \ set(email_notifications_config['public_flightreport_bad']) send_flightreport_email( list(destinations), full_plot_url, DBData.rating_str_static(rating), DBData.wind_speed_str_static(wind_speed), delete_url, stored_email, info) # also generate the additional DB entry # (we may have the log already loaded in 'ulog', however the # lru cache will make it very quick to load it again) generate_db_data_from_log_file(log_id, con) # also generate the preview image IOLoop.instance().add_callback(generate_overview_img_from_id, log_id) con.commit() cur.close() con.close() # send notification emails send_notification_email(email, full_plot_url, delete_url, info) # do not redirect for QGC if source != 'QGroundControl': self.redirect(url) except CustomHTTPError: raise except ULogException: raise CustomHTTPError( 400, 'Failed to parse the file. It is most likely corrupt.') except: print('Error when handling POST data', sys.exc_info()[0], sys.exc_info()[1]) raise CustomHTTPError(500) finally: self.multipart_streamer.release_parts()
def commit( self, backup=False, startup=False ): """ Move all written but uncommitted data into place. Return True on success Return False on error (in which case the caller should rollback()) It is safe to call this method repeatedly until it returns True. """ tmp_db_filename = config.get_db_filename() + ".tmp" tmp_snapshot_filename = config.get_snapshots_filename() + ".tmp" tmp_lastblock_filename = config.get_lastblock_filename() + ".tmp" if not os.path.exists( tmp_lastblock_filename ) and (os.path.exists(tmp_db_filename) or os.path.exists(tmp_snapshot_filename)): # we did not successfully stage the write. # rollback log.error("Partial write detected. Not committing.") return False # basic sanity checks: don't overwrite the db if the file is zero bytes, or if we can't load it if os.path.exists( tmp_db_filename ): sb = os.stat( tmp_db_filename ) if sb.st_size == 0: log.error("Partial write detected: tried to overwrite with zero-sized db! Will rollback.") return False if startup: # make sure we can load this try: with open(tmp_snapshot_filename, "r") as f: db_txt = f.read() db_json = json.loads(db_txt) except: log.error("Partial write detected: corrupt partially-committed db! Will rollback.") return False backup_time = int(time.time() * 1000000) for tmp_filename, filename in zip( [tmp_lastblock_filename, tmp_snapshot_filename, tmp_db_filename], \ [config.get_lastblock_filename(), config.get_snapshots_filename(), config.get_db_filename()] ): if not os.path.exists( tmp_filename ): continue # commit our new lastblock, consensus hash set, and state engine data try: # NOTE: rename fails on Windows if the destination exists if sys.platform == 'win32' and os.path.exists( filename ): try: os.unlink( filename ) except: pass if not backup: os.rename( tmp_filename, filename ) else: shutil.copy( tmp_filename, filename ) os.rename( tmp_filename, tmp_filename + (".%s" % backup_time)) except Exception, e: log.exception(e) return False
def get_readonly_db_state(disposition=DISPOSITION_RO): impl = config.get_implementation() db_filename = config.get_db_filename(impl=impl) db_inst = ZonefileManageDB(db_filename, disposition) return db_inst
def get(self, *args, **kwargs): search_str = self.get_argument('search[value]', '').lower() data_start = int(self.get_argument('start')) data_length = int(self.get_argument('length')) draw_counter = int(self.get_argument('draw')) json_output = dict() json_output['draw'] = draw_counter # get the logs (but only the public ones) con = sqlite3.connect(get_db_filename(), detect_types=sqlite3.PARSE_DECLTYPES) cur = con.cursor() cur.execute( 'SELECT Id, Date, Description, WindSpeed, Rating, VideoUrl ' 'FROM Logs WHERE Public = 1 ORDER BY Date DESC') def get_columns_from_tuple(db_tuple, counter): """ load the columns (list of strings) from a db_tuple """ db_data = DBData() log_id = db_tuple[0] log_date = db_tuple[1].strftime('%Y-%m-%d') db_data.description = db_tuple[2] db_data.feedback = '' db_data.type = '' db_data.wind_speed = db_tuple[3] db_data.rating = db_tuple[4] db_data.video_url = db_tuple[5] db_data_gen = get_generated_db_data_from_log(log_id, con, cur) if db_data_gen is None: return None # bring it into displayable form ver_sw = db_data_gen.ver_sw if len(ver_sw) > 10: ver_sw = ver_sw[:6] if len(db_data_gen.ver_sw_release) > 0: try: release_split = db_data_gen.ver_sw_release.split() release_type = int(release_split[1]) if release_type == 255: # it's a release ver_sw = release_split[0] except: pass airframe_data = get_airframe_data(db_data_gen.sys_autostart_id) if airframe_data is None: airframe = db_data_gen.sys_autostart_id else: airframe = airframe_data['name'] flight_modes = ', '.join([ flight_modes_table[x][0] for x in db_data_gen.flight_modes if x in flight_modes_table ]) m, s = divmod(db_data_gen.duration_s, 60) h, m = divmod(m, 60) duration_str = '{:d}:{:02d}:{:02d}'.format(h, m, s) # make sure to break long descriptions w/o spaces (otherwise they # mess up the layout) description = html_long_word_force_break(db_data.description) return [ counter, '<a href="plot_app?log=' + log_id + '">' + log_date + '</a>', description, db_data_gen.mav_type, airframe, db_data_gen.sys_hw, ver_sw, duration_str, db_data.rating_str(), db_data_gen.num_logged_errors, flight_modes ] # need to fetch all here, because we will do more SQL calls while # iterating (having multiple cursor's does not seem to work) db_tuples = cur.fetchall() json_output['recordsTotal'] = len(db_tuples) json_output['data'] = [] if data_length == -1: data_length = len(db_tuples) filtered_counter = 0 if search_str == '': # speed-up the request by iterating only over the requested items counter = len(db_tuples) - data_start + 1 for i in range(data_start, min(data_start + data_length, len(db_tuples))): counter -= 1 columns = get_columns_from_tuple(db_tuples[i], counter) if columns is None: continue json_output['data'].append(columns) filtered_counter = len(db_tuples) else: counter = len(db_tuples) + 1 for db_tuple in db_tuples: counter -= 1 columns = get_columns_from_tuple(db_tuple, counter) if columns is None: continue if any( [search_str in str(column).lower() for column in columns]): if filtered_counter >= data_start and \ filtered_counter < data_start + data_length: json_output['data'].append(columns) filtered_counter += 1 cur.close() con.close() json_output['recordsFiltered'] = filtered_counter self.set_header('Content-Type', 'application/json') self.write(json.dumps(json_output))
}), (r'/overview_img/(.*)', StaticFileHandler, { 'path': get_overview_img_filepath() }), ] # TODO: DON'T DO THIS def _move_file_monkeypatch(self, path): shutil.copy(self.name, path) if args.bulkupload: folder_path = os.path.abspath(args.bulkupload) if os.path.isdir(folder_path): con = sqlite3.connect(get_db_filename()) cur = con.cursor() for root, dirs, files in os.walk(folder_path): for file_name in files: file_path = os.path.join(root, file_name) with open(file_path, 'r') as file: # TODO: do actual validation here, don't just check filename _, ext = os.path.splitext(file_name) if ext not in ['.ulg', '.ulog']: print(f'Skipping non-ULog file {file_path}') continue # TODO: PLEASE don't do this, make save_uploaded_log work with real file-like objects file.move = types.MethodType(_move_file_monkeypatch, file) file.get_filename = types.MethodType( lambda self: file_name, file) formdict = {}
# make sure we can load this try: with open(tmp_snapshot_filename, "r") as f: db_txt = f.read() db_json = json.loads(db_txt) except: log.error("Partial write detected: corrupt partially-committed db! Will rollback.") return False backup_time = int(time.time() * 1000000) listing = [] listing.append(("lastblock", tmp_lastblock_filename, config.get_lastblock_filename(impl=self.impl))) listing.append(("snapshots", tmp_snapshot_filename, config.get_snapshots_filename(impl=self.impl))) listing.append(("db", tmp_db_filename, config.get_db_filename(impl=self.impl))) for i in xrange(0, len(listing)): file_type, tmp_filename, filename = listing[i] dir_path = os.path.dirname(tmp_filename) dirfd = None try: dirfd = os.open(dir_path, os.O_DIRECTORY) os.fsync(dirfd) except Exception, e: log.exception(e) log.error("FATAL: failed to sync directory %s" % dir_path) traceback.print_stack() os.abort()
def get(self, *args, **kwargs): """ GET request """ search_str = self.get_argument('search[value]', '').lower() order_ind = int(self.get_argument('order[0][column]')) order_dir = self.get_argument('order[0][dir]', '').lower() data_start = int(self.get_argument('start')) data_length = int(self.get_argument('length')) draw_counter = int(self.get_argument('draw')) sim = False real = False if search_str == "sim": print("Search string is sim") sim = True if search_str == "real": print("Search string is real") real = True json_output = dict() json_output['draw'] = draw_counter # get the logs (but only the public ones) con = sqlite3.connect(get_db_filename(), detect_types=sqlite3.PARSE_DECLTYPES) cur = con.cursor() sql_order = ' ORDER BY Date DESC' ordering_col = [ '', #table row number 'Logs.Date', '', #Overview - img 'Logs.Description', 'LogsGenerated.MavType', '', #Airframe - not from DB 'LogsGenerated.Hardware', 'LogsGenerated.Software', 'LogsGenerated.Duration', 'LogsGenerated.StartTime', '', #Rating 'LogsGenerated.NumLoggedErrors', '' #FlightModes ] if ordering_col[order_ind] != '': sql_order = ' ORDER BY ' + ordering_col[order_ind] if order_dir == 'desc': sql_order += ' DESC' cur.execute( 'SELECT Logs.Id, Logs.Date, ' ' Logs.Description, Logs.WindSpeed, ' ' Logs.Rating, Logs.VideoUrl, ' ' LogsGenerated.* ' 'FROM Logs ' ' LEFT JOIN LogsGenerated on Logs.Id=LogsGenerated.Id ' # 'WHERE Logs.Public = 1 AND NOT Logs.Source = "CI" ' # this is commented out so both public and private show + sql_order) # pylint: disable=invalid-name Columns = collections.namedtuple("Columns", "columns search_only_columns") def get_columns_from_tuple(db_tuple, counter): """ load the columns (list of strings) from a db_tuple """ db_data = DBDataJoin() log_id = db_tuple[0] log_date = db_tuple[1].strftime('%Y-%m-%d') db_data.description = db_tuple[2] db_data.feedback = '' db_data.type = '' db_data.wind_speed = db_tuple[3] db_data.rating = db_tuple[4] db_data.video_url = db_tuple[5] generateddata_log_id = db_tuple[6] if log_id != generateddata_log_id: print('Join failed, loading and updating data') db_data_gen = get_generated_db_data_from_log(log_id, con, cur) if db_data_gen is None: return None db_data.add_generated_db_data_from_log(db_data_gen) else: db_data.duration_s = db_tuple[7] db_data.mav_type = db_tuple[8] db_data.estimator = db_tuple[9] db_data.sys_autostart_id = db_tuple[10] db_data.sys_hw = db_tuple[11] db_data.ver_sw = db_tuple[12] db_data.num_logged_errors = db_tuple[13] db_data.num_logged_warnings = db_tuple[14] db_data.flight_modes = \ {int(x) for x in db_tuple[15].split(',') if len(x) > 0} db_data.ver_sw_release = db_tuple[16] db_data.vehicle_uuid = db_tuple[17] db_data.flight_mode_durations = \ [tuple(map(int, x.split(':'))) for x in db_tuple[18].split(',') if len(x) > 0] db_data.start_time_utc = db_tuple[19] # bring it into displayable form ver_sw = db_data.ver_sw if len(ver_sw) > 10: ver_sw = ver_sw[:6] if len(db_data.ver_sw_release) > 0: try: release_split = db_data.ver_sw_release.split() release_type = int(release_split[1]) if release_type == 255: # it's a release ver_sw = release_split[0] except: pass airframe_data = get_airframe_data(db_data.sys_autostart_id) if airframe_data is None: airframe = db_data.sys_autostart_id else: airframe = airframe_data['name'] flight_modes = ', '.join([ flight_modes_table[x][0] for x in db_data.flight_modes if x in flight_modes_table ]) m, s = divmod(db_data.duration_s, 60) h, m = divmod(m, 60) duration_str = '{:d}:{:02d}:{:02d}'.format(h, m, s) start_time_str = 'N/A' if db_data.start_time_utc != 0: start_datetime = datetime.fromtimestamp(db_data.start_time_utc) start_time_str = start_datetime.strftime("%Y-%m-%d %H:%M") # make sure to break long descriptions w/o spaces (otherwise they # mess up the layout) description = html_long_word_force_break(db_data.description) search_only_columns = [] if db_data.ver_sw is not None: search_only_columns.append(db_data.ver_sw) if db_data.ver_sw_release is not None: search_only_columns.append(db_data.ver_sw_release) if db_data.vehicle_uuid is not None: search_only_columns.append(db_data.vehicle_uuid) image_col = '<div class="no_map_overview"> Not rendered / No GPS </div>' image_filename = os.path.join(get_overview_img_filepath(), log_id + '.png') if os.path.exists(image_filename): image_col = '<img class="map_overview" src="/overview_img/' image_col += log_id + '.png" alt="Overview Image Load Failed" height=50/>' if sim: templog_id = log_id + "sim" elif real: templog_id = log_id + "real" else: templog_id = log_id return Columns([ counter, '<a href="thiel_app?log=' + templog_id + 'desc:' + description + '">' + log_date + '</a>', image_col, description, db_data.mav_type, airframe, db_data.sys_hw, ver_sw, duration_str, start_time_str, db_data.rating_str(), db_data.num_logged_errors, flight_modes ], search_only_columns) # need to fetch all here, because we will do more SQL calls while # iterating (having multiple cursor's does not seem to work) db_tuples = cur.fetchall() json_output['recordsTotal'] = len(db_tuples) json_output['data'] = [] if data_length == -1: data_length = len(db_tuples) filtered_counter = 0 if search_str == '': # speed-up the request by iterating only over the requested items counter = data_start for i in range(data_start, min(data_start + data_length, len(db_tuples))): counter += 1 columns = get_columns_from_tuple(db_tuples[i], counter) if columns is None: continue json_output['data'].append(columns.columns) filtered_counter = len(db_tuples) else: counter = 1 for db_tuple in db_tuples: counter += 1 columns = get_columns_from_tuple(db_tuple, counter) if columns is None: continue if any([search_str in str(column).lower() for column in \ (columns.columns, columns.search_only_columns)]): if data_start <= filtered_counter < data_start + data_length: json_output['data'].append(columns.columns) filtered_counter += 1 cur.close() con.close() json_output['recordsFiltered'] = filtered_counter self.set_header('Content-Type', 'application/json') self.write(json.dumps(json_output))
def __init__(self, plot_config, verbose_output=False): self._config = plot_config self._verbose_output = verbose_output # lists of dates when a _log was uploaded, one list per type self._public_logs_dates = [] self._web_ui_logs_dates = [] self._qgc_logs_dates = [] # private uploads via QGC self._ci_logs_dates = [] self._all_logs_dates = [] self._public_logs = [] # read from the DB con = sqlite3.connect(get_db_filename(), detect_types=sqlite3.PARSE_DECLTYPES) with con: cur = con.cursor() cur.execute('select Id, Date, Source, Public, Rating from Logs') db_tuples = cur.fetchall() for db_tuple in db_tuples: log = _Log(db_tuple) self._all_logs_dates.append(log.date) if log.is_public == 1: self._public_logs_dates.append(log.date) else: if log.source == 'CI': self._ci_logs_dates.append(log.date) elif log.source == 'QGroundControl': self._qgc_logs_dates.append(log.date) else: self._web_ui_logs_dates.append(log.date) # LogsGenerated: public only if log.is_public != 1: continue cur.execute('select * from LogsGenerated where Id = ?', [log.log_id]) db_tuple = cur.fetchone() if db_tuple is None: print("Error: no generated data") continue log.set_generated(db_tuple) # filter bogus entries if log.sw_version == 'v0.0.0': if self._verbose_output: print('Warning: %s with version=v0.0.0' % log.log_id) continue if log.sw_version == '': # FIXME: does that still occur and if so why? if self._verbose_output: print('Warning: %s version not set' % log.log_id) continue if log.autostart_id == 0: print('Warning: %s with autostart_id=0' % log.log_id) continue self._public_logs.append(log) self._version_data = {} # dict of _VersionData items self._all_airframes = set() self._all_boards = set() self._all_ratings = set() self._all_flight_modes = set() self._total_duration = 0 # in hours, public logs only for log in self._public_logs: if not log.sw_version in self._version_data: self._version_data[log.sw_version] = _VersionData() self._all_airframes.add(str(log.autostart_id)) self._all_boards.add(log.hardware) self._all_ratings.add(log.rating) cur_version_data = self._version_data[log.sw_version] boards = cur_version_data.boards airframes = cur_version_data.airframes airframes_num_logs = cur_version_data.airframes_num_logs ratings = cur_version_data.ratings flight_modes = cur_version_data.flight_mode_durations if not log.hardware in boards: boards[log.hardware] = 0 boards[log.hardware] += log.duration / 3600. for flight_mode, duration in log.flight_mode_durations: flight_mode_str = str(flight_mode) self._all_flight_modes.add(flight_mode_str) if not flight_mode_str in flight_modes: flight_modes[flight_mode_str] = 0. flight_modes[flight_mode_str] += duration / 3600. autostart_str = str(log.autostart_id) if not autostart_str in airframes: airframes[autostart_str] = 0 airframes_num_logs[autostart_str] = 0 airframes[autostart_str] += log.duration / 3600. airframes_num_logs[autostart_str] += 1 if not log.rating in ratings: ratings[log.rating] = 0 ratings[log.rating] += 1 self._total_duration += log.duration / 3600.
try: with open(tmp_snapshot_filename, "r") as f: db_txt = f.read() db_json = json.loads(db_txt) except: log.error("Partial write detected: corrupt partially-committed db! Will rollback.") return False backup_time = int(time.time() * 1000000) listing = [] listing.append( ("lastblock", tmp_lastblock_filename, config.get_lastblock_filename(impl=self.impl)) ) listing.append( ("snapshots", tmp_snapshot_filename, config.get_snapshots_filename(impl=self.impl)) ) listing.append( ("db", tmp_db_filename, config.get_db_filename(impl=self.impl)) ) for i in xrange(0, len(listing)): file_type, tmp_filename, filename = listing[i] dir_path = os.path.dirname( tmp_filename ) dirfd = None try: dirfd = os.open(dir_path, os.O_DIRECTORY) os.fsync(dirfd) except Exception, e: log.exception(e) log.error("FATAL: failed to sync directory %s" % dir_path) traceback.print_stack() os.abort()