def test_storage_create_path_does_not_raise_exception_if_path_already_exists( self): Storage(StubParser(), StubIO(), './bar') self.assertTrue(os.path.isdir('./bar')) Storage(StubParser(), StubIO(), './bar') self.assertTrue(os.path.isdir('./bar')) os.rmdir('./bar')
def test_storage_get_fetches_from_storage_correctly(self): storage = Storage(StubParser(), StubIO(), '.') storage.io.file['./name.stub'] = 'foo' tasks = storage.get('name') self.assertTrue(isinstance(tasks, Tasks)) self.assertEqual('Stub Job 1', tasks.all()[0].name) self.assertEqual('Stub Job 2', tasks.all()[1].name)
def _get_absolute_file_path(self, query): filename = "query-result-{}".format(str(uuid.uuid4())[:8]) relative_dir = "org_{0}_proj_{1}".format( query.parent_organization.slug, query.parent_project.id) relative_path = os.path.join(relative_dir, filename) absolute_path = Storage().get_path(relative_dir, filename) print("result:", absolute_path) return absolute_path
def __init__(self): asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) self.executor = ThreadPoolExecutor(max_workers=2) self.storage = Storage() self.api = API(self.storage, self.executor) self.task_processor = TaskProcessor(self.storage) self.api.set_execute_and_callback(self.task_processor.execute, self.task_processor.callback)
def connect(self): """ Connects to a storage :return: """ storage_type = self.config.get('cache', 'type') self.__log.info( "Creating the storage cache of type {}".format(storage_type)) cache = Storage(storage_type, self.config) #.cache self.__log.info("Connected to cache") return cache
def put(self, request, organization_slug, relative_dir, filename, format=None): file_obj = request.data["file"] storage = Storage() absolute_path = storage.get_path(relative_dir, filename) with open(absolute_path, "wb+") as fout: for chunk in file_obj.chunks(): fout.write(chunk) return Response(status=status.HTTP_201_CREATED)
def main(): parser = argparse.ArgumentParser() parser.add_argument("-d", "--date", dest="date") parser.add_argument("-u", "--url", dest="url") args = parser.parse_args() date = Date.parser(os.environ.get('DATE', args.date)) url = Url.stringify(os.environ.get('URL', args.url), date.year, date.month, date.day) download = Download() with Storage(date) as storage: for hour in range(1, 25): storage.feed(hour, download.json(Url.stringify(url, hour)))
def run(self): # read data logger.info("ComputeBatchPrediction::run") batch = MLBatchPrediction.objects.get(pk=self.job_params.get("db_id")) logger.info("batch", batch) # {'parent_mlmodel': 9, 'parent_dataframe': 1, 'db_id': 1, 'created_by_id': 1, 'parent_organization_id': 1, 'parent_project_id': 1} mlmodel = MLModel.objects.get(pk=self.job_params.get("parent_mlmodel")) logger.info(mlmodel.save_details) logger.info(mlmodel.all_params) il = IterativeLearner(mlmodel.all_params) il.load(mlmodel.save_details) logger.info(batch.parent_dataframe.absolute_path) input_df = DataServe.get(batch.parent_dataframe.absolute_path) predictions = il.predict(input_df) logger.info(predictions) filename = "predictions-{0}.csv".format(str(uuid.uuid4())[:8]) organization_slug = batch.parent_organization.slug project_id = batch.parent_project.id relative_dir = "org_{0}_proj_{1}".format(organization_slug, project_id) relative_path = os.path.join(relative_dir, filename) result_absolute_path = Storage().get_path(relative_dir, filename) logger.info(result_absolute_path) df = pd.DataFrame({"prediction": predictions}) df.to_csv(result_absolute_path, index=False) # create mljar data frame result_df = DataFrame( source_id=self.job_params.get("parent_dataframe"), # fix this absolute_path=result_absolute_path, file_size=1, # TODO fix the file size columns_details="", # we can describe any data frame (always :-)) preview_absolute_path="", created_by_id=self.job_params["created_by_id"], parent_organization_id=self.job_params["parent_organization_id"], parent_project_id=self.job_params["parent_project_id"], ) result_df.save() batch.result_dataframe = result_df batch.status = "done" batch.save()
def __init__(self): # получаю доступ к вк vk = Storage(login, password, 'VK').get() r, msg = vk.login() if r is False: print msg sys.exit(1) # устанавливаю плеер, запускается в отдельном потоке self.play = PlayerApp() # стартую ui self.ca = CursesApplication(self.play, vk) # стартую отдельный поток, для генерации событий, т.к. не работают с ncurses сигналы. ХЗ почему. Странно очень clock = ClockThread(self.ca) clock.start()
def get(self, request, organization_slug, project_id, filename, format=None): filename = "input-{0}-{1}".format(filename, str(uuid.uuid4())[:8]) relative_dir = "org_{0}_proj_{1}".format(organization_slug, project_id) relative_path = os.path.join(relative_dir, filename) absolute_path = Storage().get_path(relative_dir, filename) return Response({ "filename": filename, "relative_dir": relative_dir, "relative_path": relative_path, "absolute_path": absolute_path, "storage_type": "basic", })
def __init__(self, arg, author): print(author) self.msg = "" self.id_nb = 0 self.version = "" self.labels = [] self.assignee_to = [] self.author = [author] self.date_of_creation = {} self.status = status["undifined"] self.date_of_closing = {} s = "" for el in arg: s += str(el) + " " print(s) self._parse_str(s) self.set_date_of_creation() self.set_status("open") _task = self.get_task_as_dictionary() st = Storage() st.push_task(_task)
async def show_table(ctx, *, arg): print(ctx, arg) if len(arg) == 0: await ctx.send("I need a name of the table") table_name = "" for w in arg: if w == " " or w == " ": break table_name += str(w) print("table name ",table_name) st = Storage() unformated_tasks = st.get_table(table_name) print("unforameted tasks\n",unformated_tasks) if unformated_tasks: table = DiscordTable(unformated_tasks) rows = table.get_table() print("table after formatting\n", table ) print("row\n") print(rows) print("\n") await ctx.send(rows)
def __init__(self): # Create flask application self.app = Flask(__name__) # Load basic configuration self.conf = KaminaConfiguration( Path(os.path.abspath(__file__)).parents[1]).conf # Initialize storage engine self.storage = Storage(self.conf) # Routes definitions routes = [{ "r": "/api", "m": ["GET"], "f": self.index }, { "r": "/api/", "m": ["GET"], "f": self.index }, { "r": "/api/make_post", "m": ["POST"], "f": self.make_post }, { "r": "/api/make_response", "m": ["POST"], "f": self.make_response }, { "r": "/api/get_single_thread", "m": ["GET"], "f": self.get_single_thread }, { "r": "/api/get_all_threads", "m": ["GET"], "f": self.get_all_threads }] # Register all routes for route in routes: self.add_route(route)
def test_storage_path_gives_the_correct_path_to_the_storage_of_a_given_tasks( self): storage = Storage(StubParser(), StubIO(), '.') self.assertEqual('./foo.stub', storage.path('foo'))
def test_store_files(self): eso_file = EsoFile(self.ROOT, report_progress=False) s = Storage("test.db", echo=False) s.store_files(eso_file)
def __init__(self, directory_name): self.directory_name = directory_name self.serializer = Serialized() self.sto = Storage(directory_name) self.idx = index(self.sto) self.init_data()
def test_preprocess(self): # set user token = self.create_user_and_login(self.user1_params) organization = Organization.objects.get(slug=self.org1) user = MljarUser.objects.get(email=self.user1_params["email"]) project = Project( title="some title", description="...", created_by=user, parent_organization=organization, ) project.save() # prepare data local_full_file_path = "/tmp/example.csv" filename = "example.csv" relative_dir = "test" absolute_path = Storage().get_path(relative_dir, filename) data_to_file(example_X, example_y, absolute_path) ds = FileDataSource( title="my file", description="desc ...", absolute_path=absolute_path, file_name=filename, file_size=os.path.getsize(absolute_path), created_by=user, parent_organization=organization, parent_project=project, ) ds.save() job_params = { "absolute_path": ds.absolute_path, "file_name": ds.file_name, "db_id": ds.id, "created_by_id": ds.created_by.id, "parent_organization_id": ds.parent_organization.id, "parent_project_id": ds.parent_project.id, } self.assertEqual(DataFrame.objects.all().count(), 0) ######################################################################## # run job ######################################################################## process_file = ProcessUploadedFile(job_params) process_file.run() ######################################################################## # check if all is good self.assertEqual(DataFrame.objects.all().count(), 1) self.assertTrue(DataFrame.objects.filter(source_id=ds.id).count(), 1) dataframe = DataFrame.objects.get(source_id=ds.id) preview = self.request( method="get", endpoint="/api/v1/{0}/{1}/dataframe_preview/{2}".format( self.org1, project.id, dataframe.id), payload={}, token=token, expected_status_code=200, ) self.assertEqual(len(json.loads(preview.get("preview_data"))), 100) self.assertTrue("columns_description" in preview) self.assertTrue("nrows" in preview) self.assertTrue("ncols" in preview) frames = self.request( method="get", endpoint="/api/v1/{0}/{1}/dataframes".format( self.org1, project.id), payload={}, token=token, expected_status_code=200, ) print(frames)
def test_storage_put_adds_task_to_storage(self): storage = Storage(StubParser(), StubIO(), '.') storage.put(SimpleTasks('name')) self.assertEqual('Stub Job 1', storage.io.file['./name.stub'])
def test_storage_create_path_creates_the_path_if_it_does_not_exists(self): Storage(StubParser(), StubIO(), './foo') self.assertTrue(os.path.isdir('./foo')) os.rmdir('./foo')
def test_create_db(self): s = Storage("test.db", echo=False) self.assertTrue(database_exists("sqlite:///test.db")) s.delete_db()
def test_create_db_in_memory(self): _ = Storage(echo=False) self.assertTrue(database_exists("sqlite:///:memory:"))
from storage.storage import Storage from esofile_reader import EsoFile, Variable if __name__ == "__main__": eso_file = EsoFile(r"tests/eso_files/eplusout_all_intervals.eso", report_progress=False) storage = Storage("test.db", echo=False) # storage.store_files(eso_file) storage.fetch_variables("eplusout_all_intervals", [Variable("timestep", None, "Zone People Occupant Count", "")])
import os from storage.storage import Storage from flask import Flask, request, jsonify from zk.publisher import ServicePublisher SERVICE_PORT = int(os.environ['SERVICE_PORT']) ZK_HOSTS = os.environ['ZK_HOSTS'] app = Flask(__name__) storage = Storage(host=os.environ['DB_HOST'], port=int(os.environ['DB_PORT']), db=os.environ['DB_NAME'], user=os.environ['DB_USER'], passwd=os.environ['DB_PASS']) @app.route('/', methods=['GET']) def index(): return 'Storage web service up and running!' @app.route('/visits', methods=['GET']) def visits(): last_n = int(request.args.get('n')) return jsonify(storage.last_n_visits(last_n)) @app.route('/stats', methods=['POST']) def stats(): stats_data = request.get_json() storage.insert_statistics(stats_data)
def test_start_mlexperiment(self): token = self.create_user_and_login(self.user1_params) organization = Organization.objects.get(slug=self.org1) user = MljarUser.objects.get(email=self.user1_params["email"]) project = Project( title="some title", description="...", created_by=user, parent_organization=organization, ) project.save() # prepare data local_full_file_path = "/tmp/example.csv" filename = "example.csv" relative_dir = "test" absolute_path = Storage().get_path(relative_dir, filename) data_to_file(example_X, example_y, absolute_path) ds = FileDataSource( title="my file", description="desc ...", absolute_path=absolute_path, file_name=filename, file_size=os.path.getsize(absolute_path), created_by=user, parent_organization=organization, parent_project=project, ) ds.save() job_params = { "absolute_path": ds.absolute_path, "file_name": ds.file_name, "db_id": ds.id, "created_by_id": ds.created_by.id, "parent_organization_id": ds.parent_organization.id, "parent_project_id": ds.parent_project.id, } self.assertEqual(DataFrame.objects.all().count(), 0) process_file = ProcessUploadedFile(job_params) process_file.run() time.sleep(1) # not nice but till no websockets, let's use it self.assertEqual(DataFrame.objects.all().count(), 1) self.assertTrue(DataFrame.objects.filter(source_id=ds.id).count(), 1) mljar_df = DataFrame.objects.get(source_id=ds.id) print(mljar_df.columns_details) ### start ml experiment ### mlexperiment = MLExperiment( title="exp 1", description="na na ...", params={ "data_usage": { "train_absolute_path": mljar_df.absolute_path }, "metric": { "optimize": "logloss", "monitor": ["logloss", "auc"] }, "validation": { "validation_type": "split", "train_ratio": 0.5, "shuffle": True, }, "preprocessing": {}, }, column_usage={ "target": ["target"], "input": ["feature_{0}".format(i) for i in range(4)], }, created_by_id=user.id, parent_organization_id=organization.id, parent_project_id=project.id, ) mlexperiment.save() job_params = { "db_id": mlexperiment.id, "params": mlexperiment.params, "column_usage": mlexperiment.column_usage, "created_by_id": mlexperiment.created_by.id, "parent_organization_id": mlexperiment.parent_organization.id, "parent_project_id": mlexperiment.parent_project.id, } automl = StartMLExperiment(job_params) automl.run() mlexperiment = MLExperiment.objects.get(pk=mlexperiment.id) print(mlexperiment.status) time.sleep(1.5) mlexperiment = MLExperiment.objects.get(pk=mlexperiment.id) print(mlexperiment.status) self.assertEqual(mlexperiment.status, "done")
def test_create_db_full_path(self): path = os.path.join(self.ROOT, "test.db") s = Storage(path, echo=False) self.assertTrue(database_exists(f"sqlite:///{path}")) s.delete_db()
def setUp(self) -> None: self.storage = Storage(SimpleJsonParser(), MockIO(), '.')
def __init__(self): self.storage = Storage()
# init context _LOGGER.info('initializing context') from context.context import Context Context.initialize(CONFIG_FILE_PATH) # init endpoints _LOGGER.info('initializing endpoints') from endpoints import Endpoints Endpoints.initialize() # init firebase _LOGGER.info('initializing firebase') import firebase_admin from firebase_admin import credentials from firebase_admin import db with FIREBASE_CRED_PATH.open() as file: FIREBASE_DATA = json.loads(file.read()) cred = credentials.Certificate(FIREBASE_DATA) firebase_admin.initialize_app( cred, {const.DB_URL: Context.data()[const.FIREBASE][const.DB_URL]}) _LOGGER.info('initializing storage') from storage.storage import Storage storage = Storage() transactions_ref = db.reference('transactions') accounts_ref = db.reference('account') app.run_server(port=5050, host='0.0.0.0')