def main(): token = os.environ.get('TG_BOT_TOKEN') chat_id = os.environ.get('TG_BOT_CHAT_ID') lukoshko_dir = os.environ.get('LUKOSHKO_DIR') bot = Bot(token) updater = Updater(token, use_context=True) # dp = updater.dispatcher file_watcher = FileWatcher(bot, chat_id, 60) file_watcher.watch_dirs(glob(f'{lukoshko_dir}/*')) updater.start_polling() updater.idle()
def init_globals(masteruri): # initialize the global handler global _ssh_handler global _screen_handler global _start_handler global _name_resolution global _history global _file_watcher global _file_watcher_param _ssh_handler = SSHhandler() _screen_handler = ScreenHandler() _start_handler = StartHandler() _name_resolution = NameResolution() _history = History() _file_watcher = FileWatcher() _file_watcher_param = FileWatcher() # test where the roscore is running (local or remote) __is_local('localhost') ## fill cache return __is_local(_name_resolution.getHostname(masteruri)) ## fill cache
def initialize(self): # watch files for filename in self._filenames: fw = FileWatcher(filename) proc = Process(target=fw.watch, args=(self._log_q, self._running)) self._processes.append(proc) # aggregate statistics proc = Process(target=self.aggregate) self._processes.append(proc) # UI proc = Process(target=self._ui.run) self._processes.append(proc)
def init(self, oauth_credentials): # Initialize the database logger.info("%s: Initializing database" % self.email) self._data_init() # Create the Musicmanager logger.info("%s: Logging in to Google music" % self.email) self.mm = Musicmanager() if not self.mm.login(oauth_credentials): logger.info("%s: Error logging in to Google music" % self.email) return False # Check if we already have any watch paths configured config = self._read_config() watched_paths = config["watched_paths"] if "watched_paths" in config else [] logger.info("%s: Found previously watched paths %s" % (self.email, watched_paths)) # Create the FileWatcher logger.info("%s: Creating the FileWatcher" % (self.email)) self.fw = FileWatcher(self.email, self._finished_writing_callback, watched_paths) return True
class User: def __init__(self, email, app_data_dir): self.email = email self.app_data_dir = app_data_dir self.db_lock = threading.Lock() def init(self, oauth_credentials): # Initialize the database logger.info("%s: Initializing database" % self.email) self._data_init() # Create the Musicmanager logger.info("%s: Logging in to Google music" % self.email) self.mm = Musicmanager() if not self.mm.login(oauth_credentials): logger.info("%s: Error logging in to Google music" % self.email) return False # Check if we already have any watch paths configured config = self._read_config() watched_paths = config["watched_paths"] if "watched_paths" in config else [] logger.info("%s: Found previously watched paths %s" % (self.email, watched_paths)) # Create the FileWatcher logger.info("%s: Creating the FileWatcher" % (self.email)) self.fw = FileWatcher(self.email, self._finished_writing_callback, watched_paths) return True def logout(self): self.mm.logout() self.fw.stop_watching() def _read_config(self): return util.read_config(os.path.join(self.app_data_dir, self.email, CFG_FILE_NAME)) def _write_config(self, config): util.write_config(config, os.path.join(self.app_data_dir, self.email, CFG_FILE_NAME)) def get_watched_paths(self): logger.debug("reading config from %s" % os.path.join(self.app_data_dir, self.email)) config = self._read_config() logger.debug("read config: %s" % config) return config["watched_paths"] if "watched_paths" in config else [] def add_watch_path(self, path): # Add to file watcher self.fw.watch(path) # Add to the config config = self._read_config() if "watched_paths" not in config: config["watched_paths"] = [path] else: if path not in config["watched_paths"]: config["watched_paths"].append(path) self._write_config(config) def remove_watch_path(self, path): # Remove from the file watcher self.fw.remove_watch(path) # Remove from the config config = self._read_config() if "watched_paths" in config: if path in config["watched_paths"]: config["watched_paths"].remove(path) else: logger.info("%s trying to remove watch path %s that we weren't watching" % (self.email, path)) self._write_config(config) def set_default_action(self, default_action): config = self._read_config() config["default_action"] = default_action self._write_config(config) def get_default_action(self): config = self._read_config() return config.get("default_action", "scan_only") def scan_existing_files(self): watched_paths = self.get_watched_paths() logger.debug("%s Scanning existing files in these directories: %s" % (self.email, watched_paths)) for watched_path in watched_paths: logger.debug("Scanning existing files in %s" % watched_path) for root, subFolders, files in os.walk(watched_path): logger.debug("root: %s, subfolders: %s, files: %s" % (root, subFolders, files)) for file in files: filename, fileExtension = os.path.splitext(file) logger.debug("looking at file %s, filename = %s, file extension = %s" % (file, filename, fileExtension)) if fileExtension == ".mp3": logger.debug("Found file %s" % file); self._update_path(os.path.join(root, file), FileStatus.Scanned) logger.debug("scanning finished"); def upload_scanned(self): songs = self.get_all_songs() for song_path in songs.keys(): if songs[song_path]["status"] == FileStatus.Scanned: logger.debug("Uploading song %s" % song_path) self.upload(song_path) return def _data_init(self): with self.db_lock: con = sql.connect(os.path.join(self.app_data_dir, self.email, DB_NAME)) with con: cur = con.cursor() cur.execute('''CREATE TABLE IF NOT EXISTS songs(path TEXT PRIMARY KEY, id TEXT, status TEXT)''') def _update_path(self, path, status, id=None, override=False): logger.info("Updating path %s with id %s and status %s" % (path, id, status)) info = ((path, "" if not id else id, status) ) with self.db_lock: con = sql.connect(os.path.join(self.app_data_dir, self.email, DB_NAME)) with con: cur = con.cursor() if not override: # Check if the song is already in the data store and, if so, what its status is cur.execute('''SELECT status FROM songs WHERE path=(?)''', (path,)) res = cur.fetchone() if res: res = res[0] if res == FileStatus.Uploaded: # If it's already been uploaded, don't override that status with something else return cur.execute('''REPLACE INTO songs VALUES(?, ?, ?)''', info) def _finished_writing_callback(self, new_file_path): logger.debug("New file %s" % new_file_path) filename, file_extension = os.path.splitext(new_file_path) if file_extension != ".mp3": logger.debug("Skipping non-mp3 file") return self._update_path(new_file_path, FileStatus.Scanned) if self.get_default_action() == "auto_upload": logger.info("Uploading new file: %s" % new_file_path) self.upload(new_file_path) @staticmethod def _find_gmusic_song(scanned_song_tags, gmusic_songs): try: artist = scanned_song_tags.artist.lower() album = scanned_song_tags.album.lower() title = scanned_song_tags.title.lower() #logger.debug("Found scanned song %s - %s - %s" % (artist, album, title)) except: logger.debug("Error grabbing song meta data") return # Search for an uploaded song that matches for gmusic_song in gmusic_songs: up_artist = gmusic_song['artist'].lower() up_album = gmusic_song['album'].lower() up_title = gmusic_song['title'].lower() #logger.debug("Looking at song %s - %s - %s" % (up_artist, up_album, up_title)) if artist == up_artist and album == up_album and title == up_title: #logger.debug("Found match!") return gmusic_song return None def sync_library(self): logger.debug("Syncing") uploaded_songs = self.mm.get_all_songs() scanned_songs = self.get_all_songs() logger.debug("found %d scanned songs" % len(scanned_songs)) # Go through all songs marked 'scanned' and search for a matching that's already been uploaded. # If we find one, grab the id and mark the song as uploaded for song_path in scanned_songs.keys(): # Since we only have the path, scan its tags to get the meta data audioFile = eyed3.load(song_path) if audioFile and audioFile.tag: local_song = scanned_songs[song_path] gmusic_song = User._find_gmusic_song(audioFile.tag, uploaded_songs) # Now make sure our static is in sync, possibilities: # 1) We show it as uploaded but google doesn't -> mark it as 'scanned', remove our id # 2) Google shows it as uploaded but we don't -> mark it as 'uploaded', add the id # 3) We both show it as uploaded and the ids match -> do nothing # 4) Neither of us think it was uploaded -> do nothing # 5) Google has it but we don't at all -> TODO!! (option for download?) we'll need to detect this another way (currently searching only by scanned songs) if gmusic_song: # Google shows this song if local_song['status'] == FileStatus.Scanned: # Google shows it as uploaded but we don't. Mark it as uploaded and update the id #logger.debug("'%s - %s - %s' was already uploaded, updating its id to %s" % (gmusic_song['artist'], gmusic_song['album'], gmusic_song['title'], gmusic_song['id'])) self._update_path(song_path, FileStatus.Uploaded, gmusic_song['id'], override=True) elif local_song['status'] == FileStatus.Uploaded: # We both show it as uploaded, make sure ids match if local_song['id'] != gmusic_song['id']: #logger.debug("Ids differ! Updating to use google's id") self._update_path(song_path, FileStatus.Uploaded, gmusic_song['id'], override=True) else: pass #logger.debug("Ids match! No update needed") else: # No matching song on google found if local_song['status'] == FileStatus.Uploaded: #logger.debug("We show the song as uploaded but google doesn't, changing status to scanned and clearing id") self._update_path(song_path, FileStatus.Scanned, override=True) else: pass #logger.debug("Neither side thinks it's uploaded, no update needed") else: logger.debug("Error loading metadata for song %s" % song_path) def upload(self, file_path): uploaded, matched, not_uploaded = self.mm.upload(file_path, enable_matching=False) # async me! if uploaded: logger.info("Uploaded song %s with ID %s" % (file_path, uploaded[file_path])) self._update_path(file_path, FileStatus.Uploaded, uploaded[file_path]) if matched: logger.info("Matched song %s with ID %s" % (file_path, matched[file_path])) self._update_path(file_path, FileStatus.Uploaded, uploaded[file_path]) if not_uploaded: reason_string = not_uploaded[file_path] if "ALREADY_EXISTS" in reason_string: song_id = reason_string[reason_string.find("(") + 1 : reason_string.find(")")] logger.info("Song already exists with ID %s, updating database" % song_id) # The song ID is located within parentheses in the reason string self._update_path(file_path, FileStatus.Uploaded, song_id) else: logger.info("Unable to upload song %s because %s" % (file_path, reason_string)) self._update_path(file_path, FileStatus.Error, reason_string) def get_all_songs(self): songs = {} with self.db_lock: con = sql.connect(os.path.join(self.app_data_dir, self.email, DB_NAME)) with con: cur = con.cursor() for row in cur.execute('''SELECT * FROM songs'''): song_path = row[0] song_id = row[1] song_status = row[2] songs[song_path] = {'id': song_id, 'status': song_status} return songs
def drive(cfg, model_path=None, use_joystick=False, model_type=None, camera_type='single'): ''' Construct a working robotic vehicle from many parts. Each part runs as a job in the Vehicle loop, calling either it's run or run_threaded method depending on the constructor flag `threaded`. All parts are updated one after another at the framerate given in cfg.DRIVE_LOOP_HZ assuming each part finishes processing in a timely manner. Parts may have named outputs and inputs. The framework handles passing named outputs to parts requesting the same named input. ''' if model_type is None: model_type = "categorical" #Initialize car V = dk.vehicle.Vehicle() if camera_type == "stereo": if cfg.CAMERA_TYPE == "WEBCAM": from donkeycar.parts.camera import Webcam camA = Webcam(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, iCam=0) camB = Webcam(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, iCam=1) elif cfg.CAMERA_TYPE == "CVCAM": from donkeycar.parts.cv import CvCam camA = CvCam(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, iCam=0) camB = CvCam(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, iCam=1) else: raise (Exception("Unsupported camera type: %s" % cfg.CAMERA_TYPE)) V.add(camA, outputs=['cam/image_array_a'], threaded=True) V.add(camB, outputs=['cam/image_array_b'], threaded=True) def stereo_pair(image_a, image_b): ''' This will take the two images and combine them into a single image One in red, the other in green, and diff in blue channel. ''' if image_a is not None and image_b is not None: width, height, _ = image_a.shape grey_a = dk.utils.rgb2gray(image_a) grey_b = dk.utils.rgb2gray(image_b) grey_c = grey_a - grey_b stereo_image = np.zeros([width, height, 3], dtype=np.dtype('B')) stereo_image[..., 0] = np.reshape(grey_a, (width, height)) stereo_image[..., 1] = np.reshape(grey_b, (width, height)) stereo_image[..., 2] = np.reshape(grey_c, (width, height)) else: stereo_image = [] return np.array(stereo_image) image_sterero_pair_part = Lambda(stereo_pair) V.add(image_sterero_pair_part, inputs=['cam/image_array_a', 'cam/image_array_b'], outputs=['cam/image_array']) else: print("cfg.CAMERA_TYPE", cfg.CAMERA_TYPE) if cfg.CAMERA_TYPE == "PICAM": from donkeycar.parts.camera import PiCamera cam = PiCamera(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH) elif cfg.CAMERA_TYPE == "WEBCAM": from donkeycar.parts.camera import Webcam cam = Webcam(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH) elif cfg.CAMERA_TYPE == "CVCAM": from donkeycar.parts.cv import CvCam cam = CvCam(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH) else: raise (Exception("Unkown camera type: %s" % cfg.CAMERA_TYPE)) V.add(cam, outputs=['cam/image_array'], threaded=True) if use_joystick or cfg.USE_JOYSTICK_AS_DEFAULT: #modify max_throttle closer to 1.0 to have more power #modify steering_scale lower than 1.0 to have less responsive steering from donkeycar.parts.controller import PS3JoystickController, PS4JoystickController cont_class = PS3JoystickController if cfg.CONTROLLER_TYPE == "ps4": cont_class = PS4JoystickController ctr = cont_class(throttle_scale=cfg.JOYSTICK_MAX_THROTTLE, steering_scale=cfg.JOYSTICK_STEERING_SCALE, auto_record_on_throttle=cfg.AUTO_RECORD_ON_THROTTLE) if cfg.USE_NETWORKED_JS: from donkeycar.parts.controller import JoyStickSub netwkJs = JoyStickSub(cfg.NETWORK_JS_SERVER_IP) V.add(netwkJs, threaded=True) ctr.js = netwkJs else: #This web controller will create a web server that is capable #of managing steering, throttle, and modes, and more. ctr = LocalWebController() V.add(ctr, inputs=['cam/image_array'], outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'], threaded=True) #this throttle filter will allow one tap back for esc reverse th_filter = ThrottleFilter() V.add(th_filter, inputs=['user/throttle'], outputs=['user/throttle']) #See if we should even run the pilot module. #This is only needed because the part run_condition only accepts boolean def pilot_condition(mode): if mode == 'user': return False else: return True pilot_condition_part = Lambda(pilot_condition) V.add(pilot_condition_part, inputs=['user/mode'], outputs=['run_pilot']) def led_cond(mode, recording, num_records, behavior_state): #returns a blink rate. 0 for off. -1 for on. positive for rate. if num_records is not None and num_records % 10 == 0: if led_cond.last_num_rec_print != num_records: print("recorded", num_records, "records") led_cond.last_num_rec_print = num_records if behavior_state is not None and model_type == 'behavior': r, g, b = cfg.BEHAVIOR_LED_COLORS[behavior_state] led.set_rgb(r, g, b) return -1 #solid on if recording: return -1 #solid on elif mode == 'user': return 1 elif mode == 'local_angle': return 0.5 elif mode == 'local': return 0.1 return 0 led_cond.last_num_rec_print = 0 led_cond_part = Lambda(led_cond) V.add( led_cond_part, inputs=['user/mode', 'recording', "tub/num_records", 'behavior/state'], outputs=['led/blink_rate']) if cfg.HAVE_RGB_LED: from donkeycar.parts.led_status import RGB_LED led = RGB_LED(cfg.LED_PIN_R, cfg.LED_PIN_G, cfg.LED_PIN_B, cfg.LED_INVERT) led.set_rgb(cfg.LED_R, cfg.LED_G, cfg.LED_B) V.add(led, inputs=['led/blink_rate']) #IMU if cfg.HAVE_IMU: imu = Mpu6050() V.add(imu, outputs=[ 'imu/acl_x', 'imu/acl_y', 'imu/acl_z', 'imu/gyr_x', 'imu/gyr_y', 'imu/gyr_z' ], threaded=True) #Behavioral state if model_type == "behavior": bh = BehaviorPart(cfg.BEHAVIOR_LIST) V.add(bh, outputs=[ 'behavior/state', 'behavior/label', "behavior/one_hot_state_array" ]) try: ctr.set_button_down_trigger('L1', bh.increment_state) except: pass inputs = ['cam/image_array', "behavior/one_hot_state_array"] #IMU elif model_type == "imu": assert (cfg.HAVE_IMU) #Run the pilot if the mode is not user. inputs = [ 'cam/image_array', 'imu/acl_x', 'imu/acl_y', 'imu/acl_z', 'imu/gyr_x', 'imu/gyr_y', 'imu/gyr_z' ] else: inputs = ['cam/image_array'] def load_model(kl, model_path): start = time.time() print('loading model', model_path) kl.load(model_path) print('finished loading in %s sec.' % (str(time.time() - start))) def load_weights(kl, weights_path): start = time.time() print('loading model weights', weights_path) kl.model.load_weights(weights_path) print('finished loading in %s sec.' % (str(time.time() - start))) def load_model_json(kl, json_fnm): start = time.time() print('loading model json', json_fnm) import keras with open(json_fnm, 'r') as handle: contents = handle.read() kl.model = keras.models.model_from_json(contents) print('finished loading json in %s sec.' % (str(time.time() - start))) if model_path: #When we have a model, first create an appropriate Keras part kl = dk.utils.get_model_by_type(model_type, cfg) if '.h5' in model_path: #when we have a .h5 extension #load everything from the model file load_model(kl, model_path) def reload_model(filename): print(filename, "was changed!") load_model(kl, filename) from file_watcher import FileWatcher fw_part = FileWatcher(model_path, reload_model, wait_for_write_stop=10.0) V.add(fw_part) elif '.json' in model_path: #when we have a .json extension #load the model from their and look for a matching #.wts file with just weights load_model_json(kl, model_path) weights_path = model_path.replace('.json', '.weights') load_weights(kl, weights_path) def reload_weights(filename): print(filename, "was changed!") weights_path = filename.replace('.json', '.weights') load_weights(kl, weights_path) from donkeycar.parts.file_watcher import FileWatcher fw_part = FileWatcher(model_path, reload_weights, wait_for_write_stop=1.0) V.add(fw_part) else: #previous default behavior load_model(kl, model_path) V.add(kl, inputs=inputs, outputs=['pilot/angle', 'pilot/throttle'], run_condition='run_pilot') #Choose what inputs should change the car. def drive_mode(mode, user_angle, user_throttle, pilot_angle, pilot_throttle): if mode == 'user': return user_angle, user_throttle elif mode == 'local_angle': return pilot_angle, user_throttle else: return pilot_angle, pilot_throttle drive_mode_part = Lambda(drive_mode) V.add(drive_mode_part, inputs=[ 'user/mode', 'user/angle', 'user/throttle', 'pilot/angle', 'pilot/throttle' ], outputs=['angle', 'throttle']) #Drive train setup if cfg.DRIVE_TRAIN_TYPE == "SERVO_ESC": from donkeycar.parts.actuator import PCA9685, PWMSteering, PWMThrottle steering_controller = PCA9685(cfg.STEERING_CHANNEL, cfg.PCA9685_I2C_ADDR, busnum=cfg.PCA9685_I2C_BUSNUM) steering = PWMSteering(controller=steering_controller, left_pulse=cfg.STEERING_LEFT_PWM, right_pulse=cfg.STEERING_RIGHT_PWM) throttle_controller = PCA9685(cfg.THROTTLE_CHANNEL, cfg.PCA9685_I2C_ADDR, busnum=cfg.PCA9685_I2C_BUSNUM) throttle = PWMThrottle(controller=throttle_controller, max_pulse=cfg.THROTTLE_FORWARD_PWM, zero_pulse=cfg.THROTTLE_STOPPED_PWM, min_pulse=cfg.THROTTLE_REVERSE_PWM) V.add(steering, inputs=['angle']) V.add(throttle, inputs=['throttle']) elif cfg.DRIVE_TRAIN_TYPE == "DC_STEER_THROTTLE": from donkeycar.parts.actuator import Mini_HBridge_DC_Motor_PWM steering = Mini_HBridge_DC_Motor_PWM(cfg.HBRIDGE_PIN_LEFT, cfg.HBRIDGE_PIN_RIGHT) throttle = Mini_HBridge_DC_Motor_PWM(cfg.HBRIDGE_PIN_FWD, cfg.HBRIDGE_PIN_BWD) V.add(steering, inputs=['angle']) V.add(throttle, inputs=['throttle']) elif cfg.DRIVE_TRAIN_TYPE == "DC_TWO_WHEEL": from donkeycar.parts.actuator import TwoWheelSteeringThrottle, Mini_HBridge_DC_Motor_PWM left_motor = Mini_HBridge_DC_Motor_PWM(cfg.HBRIDGE_PIN_LEFT_FWD, cfg.HBRIDGE_PIN_LEFT_BWD) right_motor = Mini_HBridge_DC_Motor_PWM(cfg.HBRIDGE_PIN_RIGHT_FWD, cfg.HBRIDGE_PIN_RIGHT_BWD) two_wheel_control = TwoWheelSteeringThrottle() V.add(two_wheel_control, inputs=['throttle', 'angle'], outputs=['left_motor_speed', 'right_motor_speed']) V.add(left_motor, inputs=['left_motor_speed']) V.add(right_motor, inputs=['right_motor_speed']) elif cfg.DRIVE_TRAIN_TYPE == "SERVO_HBRIDGE_PWM": from donkeycar.parts.actuator import ServoBlaster, PWMSteering steering_controller = ServoBlaster(cfg.STEERING_CHANNEL) #really pin #PWM pulse values should be in the range of 100 to 200 assert (cfg.STEERING_LEFT_PWM <= 200) assert (cfg.STEERING_RIGHT_PWM <= 200) steering = PWMSteering(controller=steering_controller, left_pulse=cfg.STEERING_LEFT_PWM, right_pulse=cfg.STEERING_RIGHT_PWM) from donkeycar.parts.actuator import Mini_HBridge_DC_Motor_PWM motor = Mini_HBridge_DC_Motor_PWM(cfg.HBRIDGE_PIN_FWD, cfg.HBRIDGE_PIN_BWD) V.add(steering, inputs=['angle']) V.add(motor, inputs=["throttle"]) #add tub to save data inputs = ['cam/image_array', 'user/angle', 'user/throttle', 'user/mode'] types = ['image_array', 'float', 'float', 'str'] if cfg.TRAIN_BEHAVIORS: inputs += [ 'behavior/state', 'behavior/label', "behavior/one_hot_state_array" ] types += ['int', 'str', 'vector'] if cfg.HAVE_IMU: inputs += [ 'imu/acl_x', 'imu/acl_y', 'imu/acl_z', 'imu/gyr_x', 'imu/gyr_y', 'imu/gyr_z' ] types += ['float', 'float', 'float', 'float', 'float', 'float'] th = TubHandler(path=cfg.DATA_PATH) tub = th.new_tub_writer(inputs=inputs, types=types) V.add(tub, inputs=inputs, outputs=["tub/num_records"], run_condition='recording') if type(ctr) is LocalWebController: print("You can now go to <your pi ip address>:8887 to drive your car.") elif isinstance(ctr, JoystickController): print("You can now move your joystick to drive your car.") #tell the controller about the tub ctr.set_tub(tub) #run the vehicle for 20 seconds V.start(rate_hz=cfg.DRIVE_LOOP_HZ, max_loop_count=cfg.MAX_LOOPS)