def test_cancel_replication(): server = Server("http://test.test:5984") httpretty.register_uri( httpretty.HEAD, "http://test.test:5984/_replicator", status=200 ) global doc_exists doc_exists = True def head_test_src(request, uri, headers): if doc_exists: return 200, headers, "" else: return 404, headers, "" httpretty.register_uri( httpretty.HEAD, "http://test.test:5984/_replicator/test_src", etag="a", body=head_test_src ) def delete_test_src(request, uri, headers): global doc_exists doc_exists = False return 200, headers, "" httpretty.register_uri( httpretty.DELETE, "http://test.test:5984/_replicator/test_src", etag="a", body=delete_test_src ) assert "test_src" in server["_replicator"] server.cancel_replication("test_src") assert "test_src" not in server["_replicator"]
def test_cancel_replication(): server = Server("http://test.test:5984") httpretty.register_uri(httpretty.HEAD, "http://test.test:5984/_replicator", status=200) global doc_exists doc_exists = True def head_test_src(request, uri, headers): if doc_exists: return 200, headers, "" else: return 404, headers, "" httpretty.register_uri(httpretty.HEAD, "http://test.test:5984/_replicator/test_src", etag="a", body=head_test_src) def delete_test_src(request, uri, headers): global doc_exists doc_exists = False return 200, headers, "" httpretty.register_uri(httpretty.DELETE, "http://test.test:5984/_replicator/test_src", etag="a", body=delete_test_src) assert "test_src" in server["_replicator"] server.cancel_replication("test_src") assert "test_src" not in server["_replicator"]
def create_farm(farm_name): """ Create a farm. Creates a farm named FARM_NAME on the currently selected cloud server. You can use the `openag cloud select_farm` command to start mirroring data into it. """ utils.check_for_cloud_server() utils.check_for_cloud_user() server = Server(config["cloud_server"]["url"]) username = config["cloud_server"]["username"] password = config["cloud_server"]["password"] server.log_in(username, password) url = urljoin(server.resource.url, "_openag", "v0.1", "register_farm") status, _, content = server.resource.session.request( "POST", url, headers=server.resource.headers.copy(), body={ "name": username, "farm_name": farm_name }, credentials=(username, password)) if status != 200: raise click.ClickException( "Failed to register farm with cloud server ({}): {}".format( status, content))
def test_create_user(): server = Server("http://test.test:5984") httpretty.register_uri(httpretty.HEAD, "http://test.test:5984/_users") httpretty.register_uri( httpretty.PUT, "http://test.test:5984/_users/org.couchdb.user%3Atest", status=201) server.create_user("test", "test")
def register(username, password): """ Create a new user account. Creates a user account with the given credentials on the selected cloud server. """ check_for_cloud_server() server = Server(config["cloud_server"]["url"]) server.create_user(username, password)
def test_create_user(): server = Server("http://test.test:5984") httpretty.register_uri( httpretty.HEAD, "http://test.test:5984/_users" ) httpretty.register_uri( httpretty.PUT, "http://test.test:5984/_users/org.couchdb.user%3Atest", status=201 ) server.create_user("test", "test")
def login(username, password): """ Log into your user account """ check_for_cloud_server() old_username = config["cloud_server"]["username"] if old_username and old_username != username: raise click.ClickException( "Already logged in as user \"{}\". Run `openag cloud user logout` " "before attempting to log in as a different user".format( old_username)) server = Server(config["cloud_server"]["url"]) server.log_in(username, password) config["cloud_server"]["username"] = username config["cloud_server"]["password"] = password
def test_push_design_documents(): server = Server("http://test.test:5984") tempdir = tempfile.mkdtemp() try: test_db_path = os.path.join(tempdir, "test") os.mkdir(test_db_path) hidden_db_path = os.path.join(tempdir, ".test") os.mkdir(hidden_db_path) views_path = os.path.join(test_db_path, "views") os.mkdir(views_path) test_view_path = os.path.join(views_path, "test") os.mkdir(test_view_path) map_path = os.path.join(test_view_path, "map.js") with open(map_path, "w+") as f: f.write("test") hidden_map_path = os.path.join(test_view_path, ".test.js") with open(hidden_map_path, "w+") as f: f.write("test") httpretty.register_uri( httpretty.HEAD, "http://test.test:5984/test" ) def create_design_doc(request, uri, headers): obj = { "_id": "_design/openag", "views": { "test": { "map": "test" } } } if json.loads(request.body) == obj: return 200, headers, json.dumps({ "id": "_design/openag", "rev": "a" }) else: return 500, headers, "" httpretty.register_uri( httpretty.HEAD, "http://test.test:5984/test/_design/openag", status=404 ) httpretty.register_uri( httpretty.PUT, "http://test.test:5984/test/_design/openag", body=create_design_doc, content_type="application/json" ) server.push_design_documents(tempdir) finally: shutil.rmtree(tempdir)
def login(username, password): """ Log into your user account """ check_for_cloud_server() old_username = config["cloud_server"]["username"] if old_username and old_username != username: raise click.ClickException( "Already logged in as user \"{}\". Run `openag cloud user logout` " "before attempting to log in as a different user".format( old_username ) ) server = Server(config["cloud_server"]["url"]) server.log_in(username, password) config["cloud_server"]["username"] = username config["cloud_server"]["password"] = password
def test_replicate(): # Start a replication server = Server("http://test.test:5984") httpretty.register_uri(httpretty.HEAD, "http://test.test:5984/_replicator") httpretty.register_uri(httpretty.HEAD, "http://test.test:5984/_replicator/test", status=404) def replicate_test_src(request, uri, headers): httpretty.reset() httpretty.register_uri(httpretty.HEAD, "http://test.test:5984/_replicator") httpretty.register_uri(httpretty.HEAD, "http://test.test:5984/_replicator/test", status=200, etag="a") return 201, headers, json.dumps({ "id": "test_src", "rev": "a", "ok": True }) httpretty.register_uri(httpretty.PUT, "http://test.test:5984/_replicator/test", status=201, content_type="application/json", body=replicate_test_src) server.replicate("test", "test_src", "test_dest", continuous=True) assert "test" in server["_replicator"] # Make sure replicate is idempotent httpretty.register_uri(httpretty.PUT, "http://test.test:5984/_replicator/test_src", status=500) server.replicate("test", "test_src", "test_dest", continuous=True) assert "test" in server["_replicator"] # Cancel the replication def cancel_test_replication(request, uri, headers): httpretty.reset() httpretty.register_uri(httpretty.HEAD, "http://test.test:5984/_replicator") httpretty.register_uri(httpretty.HEAD, "http://test.test:5984/_replicator/test", status=404) return 200, headers, "" httpretty.register_uri(httpretty.DELETE, "http://test.test:5984/_replicator/test", status=200, body=cancel_test_replication) server.cancel_replication("test") assert "test" not in server["_replicator"]
def test_push_design_documents(): server = Server("http://test.test:5984") tempdir = tempfile.mkdtemp() try: test_db_path = os.path.join(tempdir, "test") os.mkdir(test_db_path) hidden_db_path = os.path.join(tempdir, ".test") os.mkdir(hidden_db_path) views_path = os.path.join(test_db_path, "views") os.mkdir(views_path) test_view_path = os.path.join(views_path, "test") os.mkdir(test_view_path) map_path = os.path.join(test_view_path, "map.js") with open(map_path, "w+") as f: f.write("test") hidden_map_path = os.path.join(test_view_path, ".test.js") with open(hidden_map_path, "w+") as f: f.write("test") httpretty.register_uri(httpretty.HEAD, "http://test.test:5984/test") def create_design_doc(request, uri, headers): obj = {"_id": "_design/openag", "views": {"test": {"map": "test"}}} if json.loads(request.body) == obj: return 200, headers, json.dumps({ "id": "_design/openag", "rev": "a" }) else: return 500, headers, "" httpretty.register_uri(httpretty.HEAD, "http://test.test:5984/test/_design/openag", status=404) httpretty.register_uri(httpretty.PUT, "http://test.test:5984/test/_design/openag", body=create_design_doc, content_type="application/json") server.push_design_documents(tempdir) finally: shutil.rmtree(tempdir)
def test_get_or_create_db(): server = Server("http://test.test:5984") httpretty.register_uri( httpretty.HEAD, "http://test.test:5984/test", status=404, body="" ) def create_test_db(request, uri, headers): httpretty.reset() httpretty.register_uri( httpretty.HEAD, "http://test.test:5984/test", status=200 ) httpretty.register_uri( httpretty.PUT, "http://test.test:5984/test", status=500 ) return 201, headers, "" httpretty.register_uri( httpretty.PUT, "http://test.test:5984/test", body=create_test_db ) assert "test" not in server test_db = server.get_or_create("test") assert "test" in server
def clear(): """ Clear all data on the local server. Useful for debugging purposed. """ utils.check_for_local_server() click.confirm( "Are you sure you want to do this? It will delete all of your data", abort=True) server = Server(config["local_server"]["url"]) for db_name in all_dbs: del server[db_name]
def list_farms(): """ List all farms you can manage. If you have selected a farm already, the name of that farm will be prefixed with an asterisk in the returned list. """ utils.check_for_cloud_server() utils.check_for_cloud_user() server = Server(config["cloud_server"]["url"]) server.log_in(config["cloud_server"]["username"], config["cloud_server"]["password"]) farms_list = server.get_user_info().get("farms", []) if not len(farms_list): raise click.ClickException( "No farms exist. Run `openag cloud create_farm` to create one") active_farm_name = config["cloud_server"]["farm_name"] for farm_name in farms_list: if farm_name == active_farm_name: click.echo("*" + farm_name) else: click.echo(farm_name)
def test_replicate(): # Start a replication server = Server("http://test.test:5984") httpretty.register_uri( httpretty.HEAD, "http://test.test:5984/_replicator" ) httpretty.register_uri( httpretty.HEAD, "http://test.test:5984/_replicator/test", status=404 ) def replicate_test_src(request, uri, headers): httpretty.reset() httpretty.register_uri( httpretty.HEAD, "http://test.test:5984/_replicator" ) httpretty.register_uri( httpretty.HEAD, "http://test.test:5984/_replicator/test", status=200, etag="a" ) return 201, headers, json.dumps({ "id": "test_src", "rev": "a", "ok": True }) httpretty.register_uri( httpretty.PUT, "http://test.test:5984/_replicator/test", status=201, content_type="application/json", body=replicate_test_src ) server.replicate("test", "test_src", "test_dest", continuous=True) assert "test" in server["_replicator"] # Make sure replicate is idempotent httpretty.register_uri( httpretty.PUT, "http://test.test:5984/_replicator/test_src", status=500 ) server.replicate("test", "test_src", "test_dest", continuous=True) assert "test" in server["_replicator"] # Cancel the replication def cancel_test_replication(request, uri, headers): httpretty.reset() httpretty.register_uri( httpretty.HEAD, "http://test.test:5984/_replicator" ) httpretty.register_uri( httpretty.HEAD, "http://test.test:5984/_replicator/test", status=404 ) return 200, headers, "" httpretty.register_uri( httpretty.DELETE, "http://test.test:5984/_replicator/test", status=200, body=cancel_test_replication ) server.cancel_replication("test") assert "test" not in server["_replicator"]
def list_farms(): """ List all farms you can manage. If you have selected a farm already, the name of that farm will be prefixed with an asterisk in the returned list. """ utils.check_for_cloud_server() utils.check_for_cloud_user() server = Server(config["cloud_server"]["url"]) server.log_in( config["cloud_server"]["username"], config["cloud_server"]["password"] ) farms_list = server.get_user_info().get("farms", []) if not len(farms_list): raise click.ClickException( "No farms exist. Run `openag cloud create_farm` to create one" ) active_farm_name = config["cloud_server"]["farm_name"] for farm_name in farms_list: if farm_name == active_farm_name: click.echo("*"+farm_name) else: click.echo(farm_name)
def test_get_or_create_db(): server = Server("http://test.test:5984") httpretty.register_uri(httpretty.HEAD, "http://test.test:5984/test", status=404, body="") def create_test_db(request, uri, headers): httpretty.reset() httpretty.register_uri(httpretty.HEAD, "http://test.test:5984/test", status=200) httpretty.register_uri(httpretty.PUT, "http://test.test:5984/test", status=500) return 201, headers, "" httpretty.register_uri(httpretty.PUT, "http://test.test:5984/test", body=create_test_db) assert "test" not in server test_db = server.get_or_create("test") assert "test" in server
def create_farm(farm_name): """ Create a farm. Creates a farm named FARM_NAME on the currently selected cloud server. You can use the `openag cloud select_farm` command to start mirroring data into it. """ utils.check_for_cloud_server() utils.check_for_cloud_user() server = Server(config["cloud_server"]["url"]) username = config["cloud_server"]["username"] password = config["cloud_server"]["password"] server.log_in(username, password) url = urljoin(server.resource.url, "_openag", "v0.1", "register_farm") status, _, content = server.resource.session.request( "POST", url, headers=server.resource.headers.copy(), body={ "name": username, "farm_name": farm_name }, credentials=(username, password) ) if status != 200: raise click.ClickException( "Failed to register farm with cloud server ({}): {}".format( status, content ) )
def update_module_types(): """ Download the repositories for all of the firmware_module_type records and update them using the `module.json` files from the repositories themselves. Currently only works for git repositories. """ local_url = config["local_server"]["url"] server = Server(local_url) db = server[FIRMWARE_MODULE_TYPE] temp_folder = mkdtemp() for _id in db: if _id.startswith("_"): continue obj = db[_id] new_obj = update_record(FirmwareModuleType(obj), temp_folder) new_obj["_rev"] = obj["_rev"] if new_obj != obj: db[_id] = new_obj rmtree(temp_folder)
def load_fixture(fixture_file): """ Populate the database from a JSON file. Reads the JSON file FIXTURE_FILE and uses it to populate the database. Fuxture files should consist of a dictionary mapping database names to arrays of objects to store in those databases. """ utils.check_for_local_server() local_url = config["local_server"]["url"] server = Server(local_url) fixture = json.load(fixture_file) for db_name, _items in fixture.items(): db = server[db_name] with click.progressbar(_items, label=db_name, length=len(_items)) as items: for item in items: item_id = item["_id"] if item_id in db: old_item = db[item_id] item["_rev"] = old_item["_rev"] if item == old_item: continue db[item_id] = item
convert_video(self.working_timelapse_filepath, self.timelapse_filepath) with open(self.timelapse_filepath, "r") as f: self.data_db.put_attachment(self.start_doc, f, TIMELAPSE_ATTACHMENT, "video/mp4") if __name__ == '__main__': db_server = cli_config["local_server"]["url"] if not db_server: raise RuntimeError("No local database specified") rospy.init_node("video_writer") try: timelapse_scaling_factor = rospy.get_param("~timelapse_scaling_factor") except KeyError: rospy.logwarn( "Timelapse scaling factor not specified. Defaulting to {}".format( DEFAULT_TIMELAPSE_SCALING_FACTOR)) timelapse_scaling_factor = DEFAULT_TIMELAPSE_SCALING_FACTOR server = Server(db_server) namespace = rospy.get_namespace() if namespace == '/': raise RuntimeError( "Video writer cannot be run in the global namespace. Please " "designate an environment for this module.") environment = namespace.split('/')[-2] for (camera_var, topic_type) in CAMERA_VARIABLES: mod = VideoWriter(server, environment, camera_var, timelapse_scaling_factor) threading.Thread(target=mod.run).start() rospy.spin()
def test_get_user_info(): server = Server("http://test.test:5984") # Try to get user info -- Should fail try: server.get_user_info() assert False, "Server.get_user_info should fail when not logged in" except RuntimeError as e: pass def on_get_session(request, uri, headers): credentials = request.headers.getheader("Authorization") if credentials.startswith("Basic "): username, password = b64decode(credentials[6:]).split(":") if username == "test" and password == "test": return 200, headers, '{"test": "test"}' return 401, headers, '{"test": "test"}' httpretty.register_uri(httpretty.GET, "http://test.test:5984/_session", body=on_get_session, content_type="application/json") server.log_in("test", "test") httpretty.register_uri(httpretty.HEAD, "http://test.test:5984/_users") httpretty.register_uri( httpretty.GET, "http://test.test:5984/_users/org.couchdb.user%3Atest", body='{"test": "test"}', content_type="application/json") res = server.get_user_info() assert server.get_user_info() == {"test": "test"} server.log_out() try: server.get_user_info() assert False, "Shouldn't be able to access the user's info when not logged in" except RuntimeError: pass
def test_get_user_info(): server = Server("http://test.test:5984") # Try to get user info -- Should fail try: server.get_user_info() assert False, "Server.get_user_info should fail when not logged in" except RuntimeError as e: pass def on_get_session(request, uri, headers): credentials = request.headers.getheader("Authorization") if credentials.startswith("Basic "): username, password = b64decode(credentials[6:]).split(":") if username == "test" and password == "test": return 200, headers, '{"test": "test"}' return 401, headers, '{"test": "test"}' httpretty.register_uri( httpretty.GET, "http://test.test:5984/_session", body=on_get_session, content_type="application/json" ) server.log_in("test", "test") httpretty.register_uri( httpretty.HEAD, "http://test.test:5984/_users" ) httpretty.register_uri( httpretty.GET, "http://test.test:5984/_users/org.couchdb.user%3Atest", body='{"test": "test"}', content_type="application/json" ) res = server.get_user_info() assert server.get_user_info() == {"test": "test"} server.log_out() try: server.get_user_info() assert False, "Shouldn't be able to access the user's info when not logged in" except RuntimeError: pass
def init(db_url, api_url): """ Initialize the database server. Sets some configuration parameters on the server, creates the necessary databases for this project, pushes design documents into those databases, and sets up replication with the cloud server if one has already been selected. """ old_db_url = config["local_server"]["url"] if old_db_url and old_db_url != db_url: raise click.ClickException( "Local database \"{}\" already initialized. Switching local " "databases is not currently supported".format(old_db_url) ) db_config = generate_config(api_url) server = Server(db_url) # Configure the CouchDB instance itself config_items = [] for section, values in db_config.items(): for param, value in values.items(): config_items.append((section, param, value)) with click.progressbar( config_items, label="Applying CouchDB configuration", length=len(config_items) ) as _config_items: for section, param, value in _config_items: url = urljoin(server.resource.url, "_config", section, param) try: current_val = server.resource.session.request( "GET", url )[2].read().strip() except ResourceNotFound: current_val = None desired_val = '"{}"'.format(value.replace('"', '\\"')) if current_val != desired_val: status = server.resource.session.request( "PUT", url, body=desired_val )[0] # Unless there is some delay between requests, CouchDB gets # sad for some reason if status != 200: click.ClickException( 'Failed to set configuration parameter "{}": {}'.format( param, res.content ) ) time.sleep(1) # Create all dbs on the server with click.progressbar( all_dbs, label="Creating databases", length=len(all_dbs) ) as _dbs: for db_name in _dbs: server.get_or_create(db_name) # Push design documents click.echo("Pushing design documents") design_path = os.path.dirname(_design.__file__) server.push_design_documents(design_path) # Set up replication if config["cloud_server"]["url"]: click.echo("Setting up replication with cloud server") utils.replicate_global_dbs(local_url=db_url) if config["cloud_server"]["farm_name"]: utils.replicate_per_farm_dbs(local_url=db_url) config["local_server"]["url"] = db_url
def run(categories, modules_file, project_dir, plugin, target, status_update_interval): """ Generate code for this project and run it """ project_dir = os.path.abspath(project_dir) # Make sure the project has been initialized pio_config = os.path.join(project_dir, "platformio.ini") if not os.path.isfile(pio_config): raise click.ClickException( "Not an OpenAg firmware project. To initialize a new project " "please use the `openag firmware init` command") firmware_types = [] firmware = [] local_server = config["local_server"]["url"] server = Server(local_server) if local_server else None modules_json = json.load(modules_file) if modules_file else {} if modules_json.get(FIRMWARE_MODULE_TYPE): for module in modules_json[FIRMWARE_MODULE_TYPE]: click.echo( "Parsing firmware module type \"{}\" from modules file".format( module["_id"])) firmware_types.append(FirmwareModuleType(module)) elif server: db = server[FIRMWARE_MODULE_TYPE] for _id in db: if _id.startswith("_"): continue click.echo( "Parsing firmware module type \"{}\" from server".format(_id)) firmware_types.append(FirmwareModuleType(db[_id])) # Check for working modules in the lib folder # Do this second so project-local values overwrite values from the server lib_path = os.path.join(project_dir, "lib") for dir_name in os.listdir(lib_path): dir_path = os.path.join(lib_path, dir_name) if not os.path.isdir(dir_path): continue config_path = os.path.join(dir_path, "module.json") if os.path.isfile(config_path): with open(config_path) as f: click.echo("Parsing firmware module type \"{}\" from lib " "folder".format(dir_name)) doc = json.load(f) if not doc.get("_id"): # Patch in id if id isn't present doc["_id"] = parent_dirname(config_path) firmware_types.append(FirmwareModuleType(doc)) # Get the list of modules if modules_json.get(FIRMWARE_MODULE): for module in modules_json[FIRMWARE_MODULE]: click.echo( "Parsing firmware module \"{}\" from modules file".format( module["_id"])) firmware.append(FirmwareModule(module)) elif server: db = server[FIRMWARE_MODULE] for _id in db: if _id.startswith("_"): continue click.echo( "Parsing firmware module \"{}\" from server".format(_id)) firmware.append(FirmwareModule(db[_id])) if len(firmware) == 0: click.echo("Warning: no modules specified for the project") module_types = index_by_id(firmware_types) modules = index_by_id(firmware) # Synthesize the module and module type dicts modules = synthesize_firmware_module_info(modules, module_types) # Update the module inputs and outputs using the categories modules = prune_unspecified_categories(modules, categories) # Generate src.ino src_dir = os.path.join(project_dir, "src") src_file_path = os.path.join(src_dir, "src.ino") # Load the plugins plugin_fns = (load_plugin(plugin_name) for plugin_name in plugin) # Run modules through each plugin plugins = [plugin_fn(modules) for plugin_fn in plugin_fns] # Generate the code codegen = CodeGen(modules=modules, plugins=plugins, status_update_interval=status_update_interval) pio_ids = (dep["id"] for dep in codegen.all_pio_dependencies()) for _id in pio_ids: subprocess.call(["platformio", "lib", "install", str(_id)]) lib_dir = os.path.join(project_dir, "lib") for dep in codegen.all_git_dependencies(): url = dep["url"] branch = dep.get("branch", "master") dep_folder_name = make_dir_name_from_url(url) dep_folder = os.path.join(lib_dir, dep_folder_name) if os.path.isdir(dep_folder): click.echo('Updating "{}"'.format(dep_folder_name)) subprocess.call(["git", "checkout", "--quiet", branch], cwd=dep_folder) subprocess.call(["git", "pull"], cwd=dep_folder) else: click.echo('Downloading "{}"'.format(dep_folder_name)) subprocess.call(["git", "clone", "-b", branch, url, dep_folder], cwd=lib_dir) with open(src_file_path, "w+") as f: codegen.write_to(f) # Compile the generated code command = ["platformio", "run"] if target: command.append("-t") command.append(target) env = os.environ.copy() build_flags = [] for c in categories: build_flags.append("-DOPENAG_CATEGORY_{}".format(c.upper())) env["PLATFORMIO_BUILD_FLAGS"] = " ".join(build_flags) if subprocess.call(command, cwd=project_dir, env=env): raise click.ClickException("Compilation failed")