def test_storage_create_path_does_not_raise_exception_if_path_already_exists(
self):
Storage(StubParser(), StubIO(), './bar')
self.assertTrue(os.path.isdir('./bar'))
Storage(StubParser(), StubIO(), './bar')
self.assertTrue(os.path.isdir('./bar'))
os.rmdir('./bar')
def test_storage_get_fetches_from_storage_correctly(self):
storage = Storage(StubParser(), StubIO(), '.')
storage.io.file['./name.stub'] = 'foo'
tasks = storage.get('name')
self.assertTrue(isinstance(tasks, Tasks))
self.assertEqual('Stub Job 1', tasks.all()[0].name)
self.assertEqual('Stub Job 2', tasks.all()[1].name)
def _get_absolute_file_path(self, query):
filename = "query-result-{}".format(str(uuid.uuid4())[:8])
relative_dir = "org_{0}_proj_{1}".format(
query.parent_organization.slug, query.parent_project.id)
relative_path = os.path.join(relative_dir, filename)
absolute_path = Storage().get_path(relative_dir, filename)
print("result:", absolute_path)
return absolute_path
def __init__(self):
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
self.executor = ThreadPoolExecutor(max_workers=2)
self.storage = Storage()
self.api = API(self.storage, self.executor)
self.task_processor = TaskProcessor(self.storage)
self.api.set_execute_and_callback(self.task_processor.execute,
self.task_processor.callback)
def connect(self):
"""
Connects to a storage
:return:
"""
storage_type = self.config.get('cache', 'type')
self.__log.info(
"Creating the storage cache of type {}".format(storage_type))
cache = Storage(storage_type, self.config) #.cache
self.__log.info("Connected to cache")
return cache
def put(self,
request,
organization_slug,
relative_dir,
filename,
format=None):
file_obj = request.data["file"]
storage = Storage()
absolute_path = storage.get_path(relative_dir, filename)
with open(absolute_path, "wb+") as fout:
for chunk in file_obj.chunks():
fout.write(chunk)
return Response(status=status.HTTP_201_CREATED)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-d", "--date", dest="date")
parser.add_argument("-u", "--url", dest="url")
args = parser.parse_args()
date = Date.parser(os.environ.get('DATE', args.date))
url = Url.stringify(os.environ.get('URL', args.url), date.year, date.month,
date.day)
download = Download()
with Storage(date) as storage:
for hour in range(1, 25):
storage.feed(hour, download.json(Url.stringify(url, hour)))
def run(self):
# read data
logger.info("ComputeBatchPrediction::run")
batch = MLBatchPrediction.objects.get(pk=self.job_params.get("db_id"))
logger.info("batch", batch)
# {'parent_mlmodel': 9, 'parent_dataframe': 1, 'db_id': 1, 'created_by_id': 1, 'parent_organization_id': 1, 'parent_project_id': 1}
mlmodel = MLModel.objects.get(pk=self.job_params.get("parent_mlmodel"))
logger.info(mlmodel.save_details)
logger.info(mlmodel.all_params)
il = IterativeLearner(mlmodel.all_params)
il.load(mlmodel.save_details)
logger.info(batch.parent_dataframe.absolute_path)
input_df = DataServe.get(batch.parent_dataframe.absolute_path)
predictions = il.predict(input_df)
logger.info(predictions)
filename = "predictions-{0}.csv".format(str(uuid.uuid4())[:8])
organization_slug = batch.parent_organization.slug
project_id = batch.parent_project.id
relative_dir = "org_{0}_proj_{1}".format(organization_slug, project_id)
relative_path = os.path.join(relative_dir, filename)
result_absolute_path = Storage().get_path(relative_dir, filename)
logger.info(result_absolute_path)
df = pd.DataFrame({"prediction": predictions})
df.to_csv(result_absolute_path, index=False)
# create mljar data frame
result_df = DataFrame(
source_id=self.job_params.get("parent_dataframe"), # fix this
absolute_path=result_absolute_path,
file_size=1, # TODO fix the file size
columns_details="", # we can describe any data frame (always :-))
preview_absolute_path="",
created_by_id=self.job_params["created_by_id"],
parent_organization_id=self.job_params["parent_organization_id"],
parent_project_id=self.job_params["parent_project_id"],
)
result_df.save()
batch.result_dataframe = result_df
batch.status = "done"
batch.save()
def __init__(self):
# получаю доступ к вк
vk = Storage(login, password, 'VK').get()
r, msg = vk.login()
if r is False:
print msg
sys.exit(1)
# устанавливаю плеер, запускается в отдельном потоке
self.play = PlayerApp()
# стартую ui
self.ca = CursesApplication(self.play, vk)
# стартую отдельный поток, для генерации событий, т.к. не работают с ncurses сигналы. ХЗ почему. Странно очень
clock = ClockThread(self.ca)
clock.start()
def get(self,
request,
organization_slug,
project_id,
filename,
format=None):
filename = "input-{0}-{1}".format(filename, str(uuid.uuid4())[:8])
relative_dir = "org_{0}_proj_{1}".format(organization_slug, project_id)
relative_path = os.path.join(relative_dir, filename)
absolute_path = Storage().get_path(relative_dir, filename)
return Response({
"filename": filename,
"relative_dir": relative_dir,
"relative_path": relative_path,
"absolute_path": absolute_path,
"storage_type": "basic",
})
def __init__(self, arg, author):
print(author)
self.msg = ""
self.id_nb = 0
self.version = ""
self.labels = []
self.assignee_to = []
self.author = [author]
self.date_of_creation = {}
self.status = status["undifined"]
self.date_of_closing = {}
s = ""
for el in arg:
s += str(el) + " "
print(s)
self._parse_str(s)
self.set_date_of_creation()
self.set_status("open")
_task = self.get_task_as_dictionary()
st = Storage()
st.push_task(_task)
async def show_table(ctx, *, arg):
print(ctx, arg)
if len(arg) == 0:
await ctx.send("I need a name of the table")
table_name = ""
for w in arg:
if w == " " or w == " ":
break
table_name += str(w)
print("table name ",table_name)
st = Storage()
unformated_tasks = st.get_table(table_name)
print("unforameted tasks\n",unformated_tasks)
if unformated_tasks:
table = DiscordTable(unformated_tasks)
rows = table.get_table()
print("table after formatting\n", table )
print("row\n")
print(rows)
print("\n")
await ctx.send(rows)
def __init__(self):
# Create flask application
self.app = Flask(__name__)
# Load basic configuration
self.conf = KaminaConfiguration(
Path(os.path.abspath(__file__)).parents[1]).conf
# Initialize storage engine
self.storage = Storage(self.conf)
# Routes definitions
routes = [{
"r": "/api",
"m": ["GET"],
"f": self.index
}, {
"r": "/api/",
"m": ["GET"],
"f": self.index
}, {
"r": "/api/make_post",
"m": ["POST"],
"f": self.make_post
}, {
"r": "/api/make_response",
"m": ["POST"],
"f": self.make_response
}, {
"r": "/api/get_single_thread",
"m": ["GET"],
"f": self.get_single_thread
}, {
"r": "/api/get_all_threads",
"m": ["GET"],
"f": self.get_all_threads
}]
# Register all routes
for route in routes:
self.add_route(route)
def test_storage_path_gives_the_correct_path_to_the_storage_of_a_given_tasks(
self):
storage = Storage(StubParser(), StubIO(), '.')
self.assertEqual('./foo.stub', storage.path('foo'))
def test_store_files(self):
eso_file = EsoFile(self.ROOT, report_progress=False)
s = Storage("test.db", echo=False)
s.store_files(eso_file)
def __init__(self, directory_name):
self.directory_name = directory_name
self.serializer = Serialized()
self.sto = Storage(directory_name)
self.idx = index(self.sto)
self.init_data()
def test_preprocess(self):
# set user
token = self.create_user_and_login(self.user1_params)
organization = Organization.objects.get(slug=self.org1)
user = MljarUser.objects.get(email=self.user1_params["email"])
project = Project(
title="some title",
description="...",
created_by=user,
parent_organization=organization,
)
project.save()
# prepare data
local_full_file_path = "/tmp/example.csv"
filename = "example.csv"
relative_dir = "test"
absolute_path = Storage().get_path(relative_dir, filename)
data_to_file(example_X, example_y, absolute_path)
ds = FileDataSource(
title="my file",
description="desc ...",
absolute_path=absolute_path,
file_name=filename,
file_size=os.path.getsize(absolute_path),
created_by=user,
parent_organization=organization,
parent_project=project,
)
ds.save()
job_params = {
"absolute_path": ds.absolute_path,
"file_name": ds.file_name,
"db_id": ds.id,
"created_by_id": ds.created_by.id,
"parent_organization_id": ds.parent_organization.id,
"parent_project_id": ds.parent_project.id,
}
self.assertEqual(DataFrame.objects.all().count(), 0)
########################################################################
# run job
########################################################################
process_file = ProcessUploadedFile(job_params)
process_file.run()
########################################################################
# check if all is good
self.assertEqual(DataFrame.objects.all().count(), 1)
self.assertTrue(DataFrame.objects.filter(source_id=ds.id).count(), 1)
dataframe = DataFrame.objects.get(source_id=ds.id)
preview = self.request(
method="get",
endpoint="/api/v1/{0}/{1}/dataframe_preview/{2}".format(
self.org1, project.id, dataframe.id),
payload={},
token=token,
expected_status_code=200,
)
self.assertEqual(len(json.loads(preview.get("preview_data"))), 100)
self.assertTrue("columns_description" in preview)
self.assertTrue("nrows" in preview)
self.assertTrue("ncols" in preview)
frames = self.request(
method="get",
endpoint="/api/v1/{0}/{1}/dataframes".format(
self.org1, project.id),
payload={},
token=token,
expected_status_code=200,
)
print(frames)
def test_storage_put_adds_task_to_storage(self):
storage = Storage(StubParser(), StubIO(), '.')
storage.put(SimpleTasks('name'))
self.assertEqual('Stub Job 1', storage.io.file['./name.stub'])
def test_storage_create_path_creates_the_path_if_it_does_not_exists(self):
Storage(StubParser(), StubIO(), './foo')
self.assertTrue(os.path.isdir('./foo'))
os.rmdir('./foo')
def test_create_db(self):
s = Storage("test.db", echo=False)
self.assertTrue(database_exists("sqlite:///test.db"))
s.delete_db()
def test_create_db_in_memory(self):
_ = Storage(echo=False)
self.assertTrue(database_exists("sqlite:///:memory:"))
from storage.storage import Storage
from esofile_reader import EsoFile, Variable
if __name__ == "__main__":
eso_file = EsoFile(r"tests/eso_files/eplusout_all_intervals.eso",
report_progress=False)
storage = Storage("test.db", echo=False)
# storage.store_files(eso_file)
storage.fetch_variables("eplusout_all_intervals",
[Variable("timestep", None, "Zone People Occupant Count", "")])
import os
from storage.storage import Storage
from flask import Flask, request, jsonify
from zk.publisher import ServicePublisher
SERVICE_PORT = int(os.environ['SERVICE_PORT'])
ZK_HOSTS = os.environ['ZK_HOSTS']
app = Flask(__name__)
storage = Storage(host=os.environ['DB_HOST'],
port=int(os.environ['DB_PORT']),
db=os.environ['DB_NAME'],
user=os.environ['DB_USER'],
passwd=os.environ['DB_PASS'])
@app.route('/', methods=['GET'])
def index():
return 'Storage web service up and running!'
@app.route('/visits', methods=['GET'])
def visits():
last_n = int(request.args.get('n'))
return jsonify(storage.last_n_visits(last_n))
@app.route('/stats', methods=['POST'])
def stats():
stats_data = request.get_json()
storage.insert_statistics(stats_data)
def test_start_mlexperiment(self):
token = self.create_user_and_login(self.user1_params)
organization = Organization.objects.get(slug=self.org1)
user = MljarUser.objects.get(email=self.user1_params["email"])
project = Project(
title="some title",
description="...",
created_by=user,
parent_organization=organization,
)
project.save()
# prepare data
local_full_file_path = "/tmp/example.csv"
filename = "example.csv"
relative_dir = "test"
absolute_path = Storage().get_path(relative_dir, filename)
data_to_file(example_X, example_y, absolute_path)
ds = FileDataSource(
title="my file",
description="desc ...",
absolute_path=absolute_path,
file_name=filename,
file_size=os.path.getsize(absolute_path),
created_by=user,
parent_organization=organization,
parent_project=project,
)
ds.save()
job_params = {
"absolute_path": ds.absolute_path,
"file_name": ds.file_name,
"db_id": ds.id,
"created_by_id": ds.created_by.id,
"parent_organization_id": ds.parent_organization.id,
"parent_project_id": ds.parent_project.id,
}
self.assertEqual(DataFrame.objects.all().count(), 0)
process_file = ProcessUploadedFile(job_params)
process_file.run()
time.sleep(1) # not nice but till no websockets, let's use it
self.assertEqual(DataFrame.objects.all().count(), 1)
self.assertTrue(DataFrame.objects.filter(source_id=ds.id).count(), 1)
mljar_df = DataFrame.objects.get(source_id=ds.id)
print(mljar_df.columns_details)
### start ml experiment ###
mlexperiment = MLExperiment(
title="exp 1",
description="na na ...",
params={
"data_usage": {
"train_absolute_path": mljar_df.absolute_path
},
"metric": {
"optimize": "logloss",
"monitor": ["logloss", "auc"]
},
"validation": {
"validation_type": "split",
"train_ratio": 0.5,
"shuffle": True,
},
"preprocessing": {},
},
column_usage={
"target": ["target"],
"input": ["feature_{0}".format(i) for i in range(4)],
},
created_by_id=user.id,
parent_organization_id=organization.id,
parent_project_id=project.id,
)
mlexperiment.save()
job_params = {
"db_id": mlexperiment.id,
"params": mlexperiment.params,
"column_usage": mlexperiment.column_usage,
"created_by_id": mlexperiment.created_by.id,
"parent_organization_id": mlexperiment.parent_organization.id,
"parent_project_id": mlexperiment.parent_project.id,
}
automl = StartMLExperiment(job_params)
automl.run()
mlexperiment = MLExperiment.objects.get(pk=mlexperiment.id)
print(mlexperiment.status)
time.sleep(1.5)
mlexperiment = MLExperiment.objects.get(pk=mlexperiment.id)
print(mlexperiment.status)
self.assertEqual(mlexperiment.status, "done")
def test_create_db_full_path(self):
path = os.path.join(self.ROOT, "test.db")
s = Storage(path, echo=False)
self.assertTrue(database_exists(f"sqlite:///{path}"))
s.delete_db()
def setUp(self) -> None:
self.storage = Storage(SimpleJsonParser(), MockIO(), '.')
def __init__(self):
self.storage = Storage()
# init context
_LOGGER.info('initializing context')
from context.context import Context
Context.initialize(CONFIG_FILE_PATH)
# init endpoints
_LOGGER.info('initializing endpoints')
from endpoints import Endpoints
Endpoints.initialize()
# init firebase
_LOGGER.info('initializing firebase')
import firebase_admin
from firebase_admin import credentials
from firebase_admin import db
with FIREBASE_CRED_PATH.open() as file:
FIREBASE_DATA = json.loads(file.read())
cred = credentials.Certificate(FIREBASE_DATA)
firebase_admin.initialize_app(
cred, {const.DB_URL: Context.data()[const.FIREBASE][const.DB_URL]})
_LOGGER.info('initializing storage')
from storage.storage import Storage
storage = Storage()
transactions_ref = db.reference('transactions')
accounts_ref = db.reference('account')
app.run_server(port=5050, host='0.0.0.0')