class AuthService:
name = "auth"
db = DatabaseSession(Base)
crypt = CryptHandler()
@rpc
def health(self, request):
return { "status": "success"}
@rpc
def login(self, user_id, password):
try:
user = self.db.query(User).filter_by(user_id=user_id).first()
if not user:
raise NotFound("User does not exist.")
if user.password != self.crypt.generate_hash(password, user.password):
raise LoginError("Password is not correct.")
return {
"status": "success",
"data": {
"user_id": user.user_id,
"auth_token": self.crypt.encode_auth_token(user.id)
}
}
except (NotFound, LoginError) as exp:
return {"status": "error", "service": self.name, "key": "Forbidden", "msg": str(exp)}
except Exception as exp:
return {"status": "error", "service": self.name, "key": "InternalServerError", "msg": str(exp)}
@rpc
def identify(self, token):
try:
user_id = self.crypt.decode_auth_token(token)
user = self.db.query(User).filter_by(id=user_id).first()
if not user:
raise NotFound("User does not exist.")
return {
"status": "success",
"data": UserSchema().dump(user).data
}
except (LoginError, InvalidSignatureError) as exp:
return {"status": "error", "service": self.name, "key": "Forbidden", "msg": str(exp)}
except Exception as exp:
return {"status": "error", "service": self.name, "key": "InternalServerError", "msg": str(exp)}
class CollegeService:
name = 'college_service'
db = DatabaseSession(DeclarativeBase)
@rpc
def ping(self):
return 'alive'
@rpc
def process(self, name):
id = str(uuid.uuid4())
temp_data = TempTable()
temp_data.name = name
temp_data.id = id
self.db.add(temp_data)
self.db.commit()
temp_datax = self.db.query(TempTable).filter_by(id=id).first()
return temp_datax.name
@rpc
def process_college(self, college_data):
college = College()
college.unitid = college_data['UNITID']
college.name = college_data['INSTNM']
college.address1 = college_data['ADDR']
college.city = college_data['CITY']
college.state = college_data['STABBR']
college.phone = college_data['GENTELE']
college.web_url = college_data['WEBADDR']
college.admissions_url = college_data['ADMINURL']
college.netprice_url = college_data['NPRICURL']
college.sector = college_data['SECTOR']
college.locale = college_data['LOCALE']
college.hbcu = college_data['HBCU']
college.latitude = college_data['LATITUDE']
college.longitude = college_data['LONGITUD']
#upsert
statement = pg_insert(College).values(
**college.as_dict()).on_conflict_do_update(
constraint='college_pkey', set_=college.props_dict())
self.db.execute(statement)
self.db.commit()
return college.name
class ProductService:
name = 'service_product'
db = DatabaseSession(DeclarativeBase)
event_dispatcher = EventDispatcher()
@rpc
def get_product(self, productId):
product = self.db.query(Product).get(productId)
if not product:
raise ProductNotFound(f'Product Id:{productId} not found!')
return ProductSchema(strict=True).dump(product)
@rpc
def create_product(self, product):
p = Product(name=product['name'],
description=product['description'],
short_description=product['short_description'],
reference=product['reference'],
warehouse_id=product['warehouse_id'])
self.db.add(p)
self.db.commit()
p = ProductSchema(strict=True).dump(p)
self.event_dispatcher('product_created', {'product': p})
return product
@rpc
def get_warehouse(self, warehouseId):
warehouse = self.db.query(Warehouse).get(warehouseId)
if not warehouse:
raise f'Warehouse Id:{warehouseId} not found!'
return ProductSchema(strict=True).dump(warehouse)
@rpc
def create_warehouse(self, warehouse):
w = Warehouse(name=warehouse['name'])
self.db.add(w)
self.db.commit()
w = WarehouseSchema(strict=True).dump(w)
self.event_dispatcher('warehouse_created', {'warehouse': w})
return warehouse
class ExampleService(object):
name = "exampleservice"
session = DatabaseSession(DeclBase)
@dummy
def write(self, value):
obj = ExampleModel(data=value)
self.session.add(obj)
self.session.commit()
return obj.id
@dummy
def read(self, id):
return self.session.query(ExampleModel).get(id).data
class ArticleService(AuthorMixin, ReviewMixin, ArticleMixin):
name = "articles"
tracer = Tracer()
db_session = DatabaseSession(DeclarativeBase)
event_dispatcher = EventDispatcher()
@http("GET", "/healthcheck")
def health_check_http(self, request):
return json.dumps(self.health_check())
@rpc
def health_check(self):
return {"status": "ok"}
class CommandNewsService:
name = 'command_stack'
dispatcher = EventDispatcher()
db = DatabaseSession(Base)
@rpc
def news_domain(self, data):
try:
data['id'] = str(uuid.uuid1())
news = NewsCommandModel(data)
self.db.add(news)
self.db.commit()
self.dispatcher('news_created', data)
return data.get('id')
except Exception as e:
self.db.rollback()
logging.error(e)
class UsersService:
name = "users"
db = DatabaseSession(Base)
@rpc
def create_user(self, user_data):
user = User(username=user_data["username"],
email=user_data["email"],
password=user_data["password"],
project=user_data["project"],
sa_token=user_data["sa_token"])
self.db.add(user)
self.db.commit()
return UserSchema().dump(user)
@rpc
def get_user(self, id):
user = self.db.query(User).get(id)
if not user:
raise NotFound("User with id {0} not found".format(id))
return UserSchema().dump(user)
@rpc
def update_user(self, id, user_data):
user = self.db.query(User).get(id)
for key, value in user_data.items():
if key == "password":
user.password = user.generate_hash(value)
else:
setattr(user, key, value)
self.db.commit()
return UserSchema().dump(user)
@rpc
def delete_user(self, id):
user = self.db.query(User).get(id)
self.db.delete(user)
self.db.commit()
class OrdersService:
name = 'orders'
log = StructlogDependency()
dispatch = EventDispatcher()
db = DatabaseSession(DeclarativeBase)
carts_rpc = RpcProxy('carts')
@rpc
def health_check(self):
self.log.info(f'orders.health:: start')
response = 'Orders service is up and running!'
self.log.info(f'orders.health:: response {response}')
self.log.info(f'orders.health:: end')
return response
@rpc
def create(self, cart_id):
self.log.info(f'orders.create:: start')
self.log.info(f'orders.create:: cart id {cart_id}')
cart = self.carts_rpc.show(cart_id)
order = Order(
total_price=cart['total_price'],
products=[
Product(
serial_number=product['id'],
title=product['title'],
description=product['description'],
price=product['price'],
quantity=product['quantity']
)
for product in cart['products']
]
)
self.db.add(order)
self.db.commit()
order = OrderSchema().dump(order)
self.log.info(f'orders.create: order {order}')
payload = {'order': order, 'cart_id': cart_id}
self.dispatch('order_created', payload)
self.log.info(f'orders.create:: end')
return order
class ContactsService:
name = 'contacts'
session = DatabaseSession(DeclarativeBase)
dispatch = EventDispatcher()
auto_crud = AutoCrudWithEvents(
session,
dispatch,
'contact',
model_cls=Contact,
get_method_name='get_contact',
create_method_name='create_contact',
update_method_name='update_contact',
list_method_name='list_contacts',
count_method_name='count_contacts',
create_event_name='contact_created',
update_event_name='contact_updated',
)
class ExampleService(object):
name = "exampleservice"
session = DatabaseSession(dec_base)
example_crud = AutoCrud('session',
model_cls=example_model,
list_method_name='_list_examplemodels',
delete_method_name=None)
@rpc
def get_examplemodel(self, id_):
""" Method should not be overwritten """
return "hello"
@rpc
def list_examplemodels(self, *args, **kwargs):
""" Enhancing default method behaviour """
results = self._list_examplemodels(*args, **kwargs)
for result in results:
result['more'] = 'data'
return results
class CommandStack:
name = 'command_stack'
dispatch = EventDispatcher()
db = DatabaseSession(Base)
@rpc
def create_user(self, data):
try:
user = UsersCommandModel(id=data['id'],
name=data['name'],
email=data['email'],
description=data['description'],
permission=data['permission'])
self.db.add(user)
self.db.commit()
data['id'] = user.id
self.dispatch('user_created', data)
return data
except Exception as e:
self.db.rollback()
return e
class StudentService:
name = 'student_service'
db = DatabaseSession(DeclarativeBase)
@rpc
def ping(self):
return 'alive'
@rpc
def process(self, name):
id = str(uuid.uuid4())
temp_data = TempTable()
temp_data.name = name
temp_data.id = id
self.db.add(temp_data)
self.db.commit()
temp_datax = self.db.query(TempTable).filter_by(id=id).first()
return temp_datax.name
class Command:
# class attributes
name = 'command_famous'
dispatch = EventDispatcher()
db = DatabaseSession(Base)
@rpc # establish RPC communication model
def add_news(self, data):
# all writing commands are executed in the same command
# because there is no longer 'update' action in Event Sourcing
try:
version = 1 # set version in case its a first record with this ID
if data.get('version'): # if version exists on instance...
version = (data.get('version') + 1) # ...bump it up
if data.get('id'): # if instance exist...
id = data.get('id') # ...get id
else:
id = self.db.execute(Sequence('news_id_seq')) # assign id if not exist
news = CommandNewsModel( # instantiate created/"updated" instance
id=id,
version=version,
title=data['title'],
content=data['content'],
author=data['author'],
published_at=data.get('published_at'),
tags=data['tags'],
)
self.db.add(news) # save record
self.db.commit()
# create new event to send info about the above to QueryStack (read DB)
data['id'] = news.id
data['version'] = news.version
self.dispatch('replicate_db_event', data) # generate new event with RPC call 'replicate db event'
return data
# if anything goes wrong rollback the previous save to command DB
except Exception as e:
self.db.rollback()
return e
class CommandStack:
name = 'command_stack'
dispatch = EventDispatcher()
db = DatabaseSession(Base)
@rpc
def create_user(self, data):
try:
id = str(uuid.uuid1())
user = UsersCommandModel(
id=id,
name=data['name'],
email=data['email'],
description=data['description'],
)
self.db.add(user)
self.db.commit()
data['id'] = user.id
self.dispatch('replicate_db_event', data)
return data
except Exception as e:
self.db.rollback()
return e
class User(object):
name = "user_service"
db = DatabaseSession(DeclarativeBase)
@rpc
def create_user(self, user_data):
if self.db.query(UserModel).filter_by(
username=user_data.get("username")).first():
raise ValueError
schemas = CreateUserSchema()
user_dict = schemas.load(user_data).data
user = schemas.make_user(user_dict)
user.password = generate_password_hash(user.password)
self.db.add(user)
self.db.commit()
return UserSchema().dump(user).data
@rpc
def login_user(self, user_data):
user = self.db.query(UserModel).filter_by(
username=user_data.get("username")).first()
if not user:
raise NotFound('Order with id {} not found'.format(
user_data.get("username")))
if not check_password_hash(user.password, user_data.get("password")):
raise ValueError
schemas = UserSchema()
return schemas.dump(user).data
@rpc
def get_user(self, id):
user = self.db.query(UserModel).get(id)
if not user:
raise ValueError
schemas = GetUserSchema()
return schemas.dump(user).data
class Command:
name = 'command_famous'
dispatch = EventDispatcher()
db = DatabaseSession(Base)
@rpc
def add_news(self, data):
try:
version = 1
if data.get('version'):
version = (data.get('version') + 1)
if data.get('id'):
id = data.get('id')
else:
id = self.db.execute(Sequence('news_id_seq'))
news = CommandNewsModel(
id=id,
version=version,
title=data['title'],
content=data['content'],
author=data['author'],
published_at=data.get('published_at'),
tags=data['tags'],
)
self.db.add(news)
self.db.commit()
data['id'] = news.id
data['version'] = news.version
self.dispatch('replicate_db_event', data)
return data
except Exception as e:
self.db.rollback()
return e
class UsersService:
name = "users"
db = DatabaseSession(Base)
crypt = CryptHandler()
@rpc
def health(self):
return {"status": "success"}
@rpc
def create_user(self, password):
try:
user_id = "user-" + self.crypt.generate_random_string(8)
project = "execution-environment" # environ.get("PROJECT") # TODO: Implement Project Service
sa_token = "" # environ.get("SERVICEACCOUNT_TOKEN") # TODO: Implement Project Service
password_hash = self.crypt.generate_hash(password)
user = User(
user_id=user_id,
password=password_hash,
project=project,
sa_token=sa_token
)
self.db.add(user)
self.db.commit()
return {
"status": "success",
"data": {
"user_id": user.user_id
}
}
except Exception as exp:
return {"status": "error", "service": self.name, "key": "InternalServerError", "msg": str(exp)}
class CommandStack:
name = 'command_stack'
dispatch = EventDispatcher()
db = DatabaseSession(Base)
@rpc
def user_domain(self, data):
try:
user = UsersCommandModel(id=data['id'],
name=data['name'],
email=data['email'],
description=data['description'],
permission=data['permission'])
self.db.add(user)
self.db.commit()
self.dispatch('user_created', data)
permission = self.db.query(PermissionsCommandModel).\
filter_by(name=data['permission']).one()
data['permission_description'] = permission.description
self.dispatch('permission_user_related', data)
except Exception as e:
self.db.rollback()
logging.error(e)
class JobService:
name = "jobs"
db = DatabaseSession(Base)
process_service = RpcProxy("processes")
data_service = RpcProxy("data")
validator = Validator()
taskparser = TaskParser()
api_connector = APIConnector()
template_controller = TemplateController()
@rpc
def health(self, request):
return {"status": "success"}
@rpc
def create_job(self, user_id, process_graph, output):
try:
processes = self.process_service.get_all_processes_full()["data"]
products = self.data_service.get_records()["data"]
self.validator.update_datasets(processes, products)
self.validator.validate_node(process_graph)
job = Job(user_id, process_graph, output)
self.db.add(job)
self.db.commit()
tasks = self.taskparser.parse_process_graph(
job.id, process_graph, processes)
for idx, task in enumerate(tasks):
self.db.add(task)
self.db.commit()
return {"status": "success", "data": JobSchema().dump(job).data}
except BadRequest as exp:
return {
"status": "error",
"service": self.name,
"key": "BadRequest",
"msg": str(exp)
}
except Exception as exp:
return {
"status": "error",
"service": self.name,
"key": "InternalServerError",
"msg": str(exp)
}
@rpc
def get_job(self, user_id, job_id):
try:
job = self.db.query(Job).filter_by(id=job_id).first()
if not job:
raise BadRequest("Job with id '{0}' does not exist.").format(
job_id)
if job.user_id != user_id:
raise Forbidden(
"You don't have the permission to access job with id '{0}'."
).format(job_id)
return {
"status": "success",
"data": JobSchemaFull().dump(job).data
}
except BadRequest:
return {
"status": "error",
"service": self.name,
"key": "BadRequest",
"msg": str(exp)
}
except Forbidden:
return {
"status": "error",
"service": self.name,
"key": "Forbidden",
"msg": str(exp)
}
except Exception as exp:
return {
"status": "error",
"service": self.name,
"key": "InternalServerError",
"msg": str(exp)
}
@rpc
def process_job(self, job_id):
try:
job = self.db.query(Job).filter_by(id=job_id).first()
tasks = self.db.query(Task).filter_by(job_id=job_id).all()
processes = self.process_service.get_all_processes_full()["data"]
tasks = sorted(tasks, key=lambda task: task.seq_num)
job.status = "running"
self.db.commit()
data_filter = tasks[0]
pvc = self.data_service.prepare_pvc(data_filter)["data"]
# TODO: Implement in Extraction Service
filter = tasks[0]
product = filter.args["product"]
start = filter.args["filter_daterange"]["from"]
end = filter.args["filter_daterange"]["to"]
left = filter.args["filter_bbox"]["left"]
right = filter.args["filter_bbox"]["right"]
top = filter.args["filter_bbox"]["top"]
bottom = filter.args["filter_bbox"]["bottom"]
srs = filter.args["filter_bbox"]["srs"]
bbox = [top, left, bottom, right]
# in_proj = Proj(init=srs)
# out_proj = Proj(init='epsg:4326')
# in_x1, in_y1 = bottom, left
# in_x2, in_y2 = top, right
# out_x1, out_y1 = transform(in_proj, out_proj, in_x1, in_y1)
# out_x2, out_y2 = transform(in_proj, out_proj, in_x2, in_y2)
# bbox = [out_x1, out_y1, out_x2, out_y2]
file_paths = self.data_service.get_records(qtype="file_paths",
qname=product,
qgeom=bbox,
qstartdate=start,
qenddate=end)["data"]
tasks[0].args["file_paths"] = file_paths
pvc = self.template_controller.create_pvc(
self.api_connector, "pvc-" + str(job.id), "storage-write",
"5Gi") # TODO: Calculate storage size and get storage class
previous_folder = None
for idx, task in enumerate(tasks):
try:
template_id = "{0}-{1}".format(job.id, task.id)
for p in processes:
if p["process_id"] == task.process_id:
process = p
config_map = self.template_controller.create_config(
self.api_connector, template_id, {
"template_id": template_id,
"last": previous_folder,
"args": task.args
})
image_name = process["process_id"].replace(
"_", "-").lower() # TODO: image name in process spec
status, log, obj_image_stream = self.template_controller.build(
self.api_connector,
template_id,
image_name,
"latest", # TODO: Implement tagging in process service
process["git_uri"],
process["git_ref"],
process["git_dir"])
status, log, metrics = self.template_controller.deploy(
self.api_connector,
template_id,
obj_image_stream,
config_map,
pvc,
"500m", # TODO: Implement Ressource Management
"1",
"256Mi",
"1Gi")
previous_folder = template_id
except APIConnectionError as exp:
task.status = exp.__str__()
self.db.commit()
pvc.delete(self.api_connector)
job.status = "finished"
self.db.commit()
except Exception as exp:
job.status = str(exp)
self.db.commit()
class OrdersService:
name = 'orders'
db = DatabaseSession(DeclarativeBase)
event_dispatcher = EventDispatcher()
products_grpc = GrpcProxy(
"http://products.examples.svc.cluster.local:50051", productsStub)
@grpc
def get_order(self, request, context):
log.debug("------- Get Order ------- %s", request)
order = self.db.query(Order).get(request.id)
if not order:
raise NotFound('Order with id {} not found'.format(request.id))
return self._order_response(order)
@grpc
def create_order(self, request, context):
order = Order(order_details=[
OrderDetail(
product_id=order_detail.product_id,
price=order_detail.price,
quantity=order_detail.quantity,
) for order_detail in request.order_details
])
log.info("------- Create Order with ------- %s", request.order_details)
self.db.add(order)
self.db.commit()
log.info("--- Created order --- %s", order)
updateInventoryRequest = UpdateInventoryRequest(
updateproductinventorydetails=[
UpdateProductInventoryRequestDetails(
id=order_details.product_id,
quantity=order_details.quantity,
) for order_details in order.order_details
])
log.info(
"------- Update Inventory of Products after Order creation ------- %s",
updateInventoryRequest)
with Client("//products.examples.svc.cluster.local:50051",
productsStub) as client:
response = client.update_products_inventory(updateInventoryRequest)
print(response)
# self.event_dispatcher('order_created', {
# 'order': order.to_dict(),
# })
return self._order_response(order)
@rpc
def update_order(self, order):
order_details = {
order_details['id']: order_details
for order_details in order['order_details']
}
order = self.db.query(Order).get(order['id'])
for order_detail in order.order_details:
order_detail.price = order_details[order_detail.id]['price']
order_detail.quantity = order_details[order_detail.id]['quantity']
self.db.commit()
return OrderSchema().dump(order).data
@rpc
def delete_order(self, order_id):
order = self.db.query(Order).get(order_id)
self.db.delete(order)
self.db.commit()
def _order_response(self, order):
return OrderResponse(
id=order.id,
order_details=[
OrderDetailResponse(
id=order_detail.id,
product_id=order_detail.product_id,
price=str(order_detail.price),
quantity=order_detail.quantity,
) for order_detail in order.order_details
],
)
class DevicesService:
name = 'devices'
db = DatabaseSession(DeclarativeBase)
event_dispatcher = EventDispatcher()
mqtt_rpc = RpcProxy('mqtt')
rules_rpc = RpcProxy('rules')
"""""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""
""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""" ""
@rpc
def info(self):
devices = self.db.query(Device).all()
return DeviceInfoSchema(many=True).dump(devices).data
@rpc
def get_devices(self):
devices = self.db.query(Device).all()
return DeviceSchema(many=True).dump(devices).data
@rpc
def get_nodes(self):
nodes = self.db.query(Node).all()
return NodeSchema(many=True).dump(nodes).data
@rpc
def get_properties(self):
properties = self.db.query(Property).all()
return PropertySchema(many=True).dump(properties).data
"""""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""
""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""" ""
@rpc
def get_device(self, content):
device_name = content['device_name']
device_model = self.db.query(Device).filter_by(
implementation_config_device_id=device_name).first()
if device_model is None:
print("Not a Device: " + str(device_name))
return False
return DeviceSchema().dump(device_model).data
@rpc
def get_node(self, content):
device_name = content['device_name']
node_name = content['node_name']
device_model = self.db.query(Device).filter_by(
implementation_config_device_id=device_name).first()
if device_model is None:
print("Not a Device: " + str(device_name))
return False
node_model = self.db.query(Node).filter_by(device=device_model,
name=node_name).first()
if node_model is None:
print("New node: " + str(device_name) + "/" + str(node_name))
return False
return NodeSchema().dump(node_model).data
@rpc
def get_property(self, content):
device_name = content['device_name']
node_name = content['node_name']
property_name = content['property_name']
device_model = self.db.query(Device).filter_by(
implementation_config_device_id=device_name).first()
if device_model is None:
print("Not a Device: " + str(device_name))
return False
node_model = self.db.query(Node).filter_by(device=device_model,
name=node_name).first()
if node_model is None:
print("Not a node: " + str(device_name) + "/" + str(node_name))
return False
property_model = self.db.query(Property).filter_by(
node=node_model, name=property_name).first()
if property_model is None:
print("Not a property: " + str(device_name) + "/" +
str(node_name) + "/" + str(property_name))
return False
return PropertySchema().dump(property_model).data
"""""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""
""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""" ""
@rpc
def get_device_attribute(self, content):
device_name = content['device_name']
attribute = content['attribute']
device_model = self.db.query(Device).filter_by(
implementation_config_device_id=device_name).first()
if device_model is None:
print("Not a Device: " + str(device_name))
return False
if hasattr(device_model, attribute):
return getattr(device_model, attribute, None)
@rpc
def get_node_attribute(self, content):
device_name = content['device_name']
node_name = content['node_name']
attribute = content['attribute']
device_model = self.db.query(Device).filter_by(
implementation_config_device_id=device_name).first()
if device_model is None:
print("Not a Device: " + str(device_name))
return False
node_model = self.db.query(Node).filter_by(device=device_model,
name=node_name).first()
if node_model is None:
print("Not a node: " + str(device_name) + "/" + str(node_name))
return False
if hasattr(node_model, attribute):
return getattr(node_model, attribute, None)
@rpc
def get_property_attribute(self, content):
device_name = content['device_name']
node_name = content['node_name']
property_name = content['property_name']
attribute = content['attribute']
device_model = self.db.query(Device).filter_by(
implementation_config_device_id=device_name).first()
if device_model is None:
print("Not a Device: " + str(device_name))
return False
node_model = self.db.query(Node).filter_by(device=device_model,
name=node_name).first()
if node_model is None:
print("Not a node: " + str(device_name) + "/" + str(node_name))
return False
property_model = self.db.query(Property).filter_by(
node=node_model, name=property_name).first()
if property_model is None:
print("Not a property: " + str(device_name) + "/" +
str(node_name) + "/" + str(property_name))
return False
if hasattr(property_model, attribute):
return getattr(property_model, attribute, None)
"""""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""
""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""" ""
@rpc
def create_device(self, content):
device_name = content['device_name']
device_model = self.db.query(Device).filter_by(
implementation_config_device_id=device_name).first()
if device_model is None:
device_model = Device(implementation_config_device_id=device_name)
print("New Device: " + str(device_name))
self.db.add(device_model)
self.db.commit()
@rpc
def create_node(self, content):
device_name = content['device_name']
node_name = content['node_name']
device_model = self.db.query(Device).filter_by(
implementation_config_device_id=device_name).first()
if device_model is None:
print("Not a Device: " + str(device_name))
device_model = Device(implementation_config_device_id=device_name)
print("New Device: " + str(device_name))
self.db.add(device_model)
self.db.commit()
node_model = self.db.query(Node).filter_by(device=device_model,
name=node_name).first()
if node_model is None:
node_model = Node(device=device_model, name=node_name)
print("New node: " + str(device_name) + "/" + str(node_name))
self.db.add(node_model)
self.db.commit()
@rpc
def create_property(self, content):
device_name = content['device_name']
node_name = content['node_name']
property_name = content['property_name']
device_model = self.db.query(Device).filter_by(
implementation_config_device_id=device_name).first()
if device_model is None:
device_model = Device(implementation_config_device_id=device_name)
self.db.add(device_model)
self.db.commit()
node_model = self.db.query(Node).filter_by(device=device_model,
name=node_name).first()
if node_model is None:
node_model = Node(device=device_model, name=node_name)
self.db.add(node_model)
self.db.commit()
property_model = self.db.query(Property).filter_by(
node=node_model, name=property_name).first()
if property_model is None:
print("New property: " + str(device_name) + "/" + str(node_name) +
"/" + str(property_name))
property_model = Property(node=node_model, name=property_name)
self.db.add(property_model)
self.db.commit()
"""""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""
""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""" ""
@rpc
def set_device_attribute(self, content):
device_name = content['device_name']
attribute = content['attribute']
payload = content['payload']
device_model = self.db.query(Device).filter_by(
implementation_config_device_id=device_name).first()
if device_model is not None:
if hasattr(device_model, attribute):
setattr(device_model, attribute, payload)
self.db.commit()
else:
print("Not a Device: " + str(device_name))
return False
@rpc
def set_node_attribute(self, content):
device_name = content['device_name']
node_name = content['node_name']
attribute = content['attribute']
payload = content['payload']
device_model = self.db.query(Device).filter_by(
implementation_config_device_id=device_name).first()
if device_model is None:
print("Not a Device: " + str(device_name))
return False
node_model = self.db.query(Node).filter_by(device=device_model,
name=node_name).first()
if node_model is not None:
if hasattr(node_model, attribute):
setattr(node_model, attribute, payload)
self.db.commit()
else:
print("Not a node: " + str(device_name) + "/" + str(node_name))
return False
@rpc
def set_property_attribute(self, content):
device_name = content['device_name']
node_name = content['node_name']
property_name = content['property_name']
attribute = content['attribute']
payload = content['payload']
device_model = self.db.query(Device).filter_by(
implementation_config_device_id=device_name).first()
if device_model is None:
print("Not a Device: " + str(device_name))
return False
node_model = self.db.query(Node).filter_by(device=device_model,
name=node_name).first()
if node_model is None:
print("Not a node: " + str(device_name) + "/" + str(node_name))
return False
property_model = self.db.query(Property).filter_by(
node=node_model, name=property_name).first()
if property_model is not None:
if hasattr(property_model, attribute):
setattr(property_model, attribute, payload)
self.db.commit()
else:
print("Not a node: " + str(device_name) + "/" + str(node_name) +
"/" + str(property_name))
return False
"""""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""
""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""" """""" ""
@rpc
def clean_database(self):
self.db.query(Property).delete()
self.db.commit()
print("Done cleaning properties")
self.db.query(Node).delete()
self.db.commit()
print("Done cleaning nodes")
self.db.query(Device).delete()
self.db.commit()
print("Done cleaning devices")
@rpc
def create_tables(self):
DeclarativeBase.metadata.create_all()
# Calculate NDVI
ndvi = (nir - red) / (nir + red)
ndvi = set_no_data(ndvi, np.nan, 2)
return ndvi
def calc_mintime(data):
return np.fmin.reduce(data)
def calc_maxtime(data):
return np.fmax.reduce(data)
if len(sys.argv) < 2:
print("Data PID needs to be passed by command line argument.")
sys.exit(-1)
data_pid = sys.argv[1]
# processing code
db = DatabaseSession(Base)
query = db.query(Query).filter_by(pid=data_pid).first()
if not query:
print("Data PID {} is not correct.".format(data_pid))
sys.exit(-1)
class ProcessesService:
"""Process discovery and management of user-defined process graphs."""
name = service_name
db = DatabaseSession(Base)
"""Database connection to processes database."""
data_service = RpcProxy("data")
"""Rpc connection to data service."""
@rpc
def get_user_defined(self, user: Dict[str, Any],
process_graph_id: str) -> dict:
"""Get the process graph using the provided process_graph_id.
Args:
user: The user object to determine whether the user can access the given process graph.
process_graph_id: The id of the process graph.
Returns:
A dictionary containing detailed information about the process graph and the request status or a serialized
service exception.
"""
try:
process_graph_id = self._back_convert_old_process_graph_ids(
process_graph_id)
process_graph = self.db.query(ProcessGraph) \
.filter_by(id_openeo=process_graph_id) \
.filter_by(process_definition=ProcessDefinitionEnum.user_defined) \
.first()
response = self._exist_and_authorize(user["id"], process_graph_id,
process_graph)
if isinstance(response, ServiceException):
return response.to_dict()
self.db.refresh(
process_graph
) # To ensure the object is always taken from the db
LOGGER.info(
f"Return user-defined ProcessGraph {process_graph_id}.")
return {
"status": "success",
"code": 200,
"data": ProcessGraphFullSchema().dump(process_graph)
}
except Exception as exp:
return ServiceException(ProcessesService.name, 500, user["id"],
str(exp)).to_dict()
@rpc
def delete(self, user: Dict[str, Any], process_graph_id: str) -> dict:
"""Delete the process graph with the given process_graph_id from the database.
Args:
user: The user object to determine whether the user can delete the given process graph.
process_graph_id: The id of the process graph.
Returns:
A dictionary with the status of the request.
"""
try:
process_graph_id = self._back_convert_old_process_graph_ids(
process_graph_id)
# Check process graph exists and user is allowed to access / delete it
process_graph = self.db.query(ProcessGraph).filter_by(
id_openeo=process_graph_id).first()
response = self._exist_and_authorize(user["id"], process_graph_id,
process_graph)
if isinstance(response, ServiceException):
return response.to_dict()
self.db.delete(process_graph)
self.db.commit()
LOGGER.info(
f"User-defined ProcessGraph {process_graph_id} successfully deleted."
)
return {"status": "success", "code": 204}
except Exception as exp:
return ServiceException(ProcessesService.name,
500,
user["id"],
str(exp),
links=[]).to_dict()
@rpc
def get_all_predefined(self, user: Dict[str, Any] = None) -> dict:
"""Return detailed process description of all available predefined processes.
Args:
user: The user object - not used, only given for consistency with all other methods.
"""
try:
process_graphs = self.db.query(ProcessGraph) \
.filter_by(process_definition=ProcessDefinitionEnum.predefined) \
.order_by(ProcessGraph.created_at).all()
LOGGER.debug(f"Found {len(process_graphs)} pre-defined processes.")
return {
"status": "success",
"code": 200,
"data": {
"processes":
ProcessGraphPredefinedSchema(
many=True).dump(process_graphs),
"links": []
}
}
except Exception as exp:
return ServiceException(ProcessesService.name, 500,
self.get_user_id(user),
str(exp)).to_dict()
@rpc
def get_all_user_defined(self, user: Dict[str, Any]) -> dict:
"""Return all available process graphs for the given user.
Args:
user: The user object.
"""
try:
process_graphs = self.db.query(ProcessGraph) \
.filter_by(process_definition=ProcessDefinitionEnum.user_defined) \
.filter_by(user_id=user["id"]) \
.order_by(ProcessGraph.created_at).all()
LOGGER.info(
f"Found {len(process_graphs)} ProcessGraphs for User {user['id']}."
)
return {
"status": "success",
"code": 200,
"data": {
"processes":
ProcessGraphShortSchema(many=True).dump(process_graphs),
"links": [],
}
}
except Exception as exp:
return ServiceException(
ProcessesService.name,
500,
user["id"],
str(exp),
links=["#tag/Job-Management/paths/~1process_graphs/get"
]).to_dict()
@rpc
def put_predefined(self,
process_name: str,
user: Dict[str, Any] = None) -> dict:
"""Add the process definition from GitHub using the process_name.
Args:
process_name: The name of the process to get from GitHub and create in the database.
user: The user object - not used, only given for consistency with all other methods.
"""
try:
process_url = f"{settings.PROCESSES_GITHUB_URL}{process_name}.json"
process_graph_response = requests.get(process_url)
if process_graph_response.status_code != 200:
return ServiceException(
ProcessesService.name, process_graph_response.status_code,
self.get_user_id(user),
str(process_graph_response.text)).to_dict()
LOGGER.debug(
f"Pre-defined Process {process_name} description is available on GitHub."
)
process_graph = self.db.query(ProcessGraph).filter_by(
id_openeo=process_name).first()
if process_graph:
self.db.delete(process_graph)
self.db.commit()
LOGGER.debug(
f"Pre-defined Process {process_name} delete to create new one with the same id."
)
process_graph_json = json.loads(process_graph_response.content)
process_graph_json[
'process_definition'] = ProcessDefinitionEnum.predefined
process_graph = ProcessGraphPredefinedSchema().load(
process_graph_json)
self.db.add(process_graph)
self.db.commit()
LOGGER.info(
f"Pre-defined Process {process_name} successfully created.")
return {
"status": "success",
"code": 201,
"headers": {
"Location": "/processes"
},
"service_data": {
'process_name': process_name
}
}
except Exception as exp:
return ServiceException(ProcessesService.name, 500,
self.get_user_id(user),
str(exp)).to_dict()
@rpc
def put_user_defined(self, user: Dict[str, Any], process_graph_id: str,
**process_graph_args: Any) -> dict:
"""Create a new user-defined process graph using the description send in the request body.
Args:
user: The user object who creates the process graph.
process_graph_id: The identifier of the process graph. This id overwrites the id in the process_graph_args
if one is given.
process_graph_args: The dictionary containing information needed to create a ProcessGraph.
Returns:
A dictionary with the status of the request.
"""
try:
response = self._check_not_in_predefined(user["id"],
process_graph_id)
if isinstance(response, ServiceException):
return response.to_dict()
process_graph_args[
'id'] = process_graph_id # path parameter overwrites id in json
validate_result = self.validate(user,
**deepcopy(process_graph_args))
if validate_result["status"] == "error":
return validate_result
process_graph = self.db.query(ProcessGraph).filter_by(
id_openeo=process_graph_id).first()
if process_graph:
response = self._authorize(user["id"], process_graph)
if isinstance(response, ServiceException):
return response.to_dict()
process_graph_args['id_internal'] = process_graph.id
process_graph_args[
'process_definition'] = ProcessDefinitionEnum.user_defined
process_graph_args['user_id'] = user["id"]
process_graph = ProcessGraphPredefinedSchema().load(
process_graph_args)
self.db.merge(process_graph)
self.db.commit()
LOGGER.info(
f"User-defined ProcessGraph {process_graph_id} successfully created."
)
return {
"status": "success",
"code": 200,
}
except Exception as exp:
return ServiceException(ProcessesService.name, 500, user["id"],
str(exp)).to_dict()
@rpc
def validate(self, user: Dict[str, Any], **process: dict) -> dict:
"""Validate the provided process graph.
Args:
user: The user object.
process: The process graph to validated.
Returns:
A dictionary including the status of the request and validation errors or a serialized service exception.
"""
# TODO: RESPONSE HEADERS -> OpenEO-Costs
try:
# Get all processes
process_response = self.get_all_predefined()
if process_response["status"] == "error":
return process_response
processes = process_response["data"]["processes"]
# Get all collections (full metadata)
data_response = self.data_service.get_all_products()
if data_response["status"] == "error":
return data_response
collections_full_md = []
for col in data_response["data"]["collections"]:
data_response = self.data_service.get_product_detail(
collection_id=col['id'])
if data_response["status"] == "error":
return data_response
collections_full_md.append(data_response['data'])
try:
_ = validate_process_graph(process,
processes_src=processes,
collections_src=collections_full_md)
output_errors: list = []
except ValidationError as exp:
output_errors = [{'code': 400, 'message': str(exp)}]
return {
"status": "success",
"code": 200,
"data": {
'errors': output_errors
}
}
except ValidationError as exp:
return ServiceException(
ProcessesService.name,
400,
user["id"],
str(exp),
internal=False,
links=["#tag/EO-Data-Discovery/paths/~1process_graph/post"
]).to_dict()
except Exception as exp:
return ServiceException(ProcessesService.name, 500, user["id"],
str(exp)).to_dict()
def _exist_and_authorize(self, user_id: str, process_graph_id: str, process_graph: ProcessGraph) \
-> Optional[ServiceException]:
"""Return Exception if given ProcessGraph does not exist or User is not allowed to access this ProcessGraph.
Args:
user_id: The identifier of the user.
process_graph_id: The id of the process graph.
process_graph: The ProcessGraph object for the given process_graph_id.
"""
exists = self._check_exists(user_id, process_graph_id, process_graph)
if isinstance(exists, ServiceException):
return exists
auth = self._authorize(user_id, process_graph)
if not isinstance(auth, ServiceException):
return auth
return None
def _check_exists(self, user_id: str, process_graph_id: str, process_graph: ProcessGraph) \
-> Optional[ServiceException]:
"""Return Exception if given ProcessGraph does not exist.
Args:
user_id The identifier of the user.
process_graph_id: The id of the process graph.
process_graph. The ProcessGraph object for the given process_graph_id.
"""
if process_graph is None:
return ServiceException(
ProcessesService.name,
400,
user_id,
f"The process_graph with id '{process_graph_id}' does not exist.",
internal=False,
links=[])
LOGGER.info(f"ProcessGraph {process_graph_id} exists.")
return None
def _authorize(self, user_id: str,
process_graph: ProcessGraph) -> Optional[ServiceException]:
"""Return Exception if the User is not allowed to access this ProcessGraph.
Args:
user_id: The identifier of the user.
process_graph: The ProcessGraph object for the given process_graph_id.
"""
if not process_graph:
return None
# TODO: Permission (e.g admin)
if process_graph.user_id != user_id:
return ServiceException(
ProcessesService.name,
401,
user_id,
"You are not allowed to access this resource.",
internal=False,
links=[])
LOGGER.info(
f"User {user_id} is authorized to access ProcessGraph {process_graph.id}"
)
return None
def _check_not_in_predefined(
self, user_id: str,
process_graph_id: str) -> Optional[ServiceException]:
"""Return Exception if the process_graph_id is defined as predefined process.
Args:
user_id: The identifier of the user.
process_graph_id: The id of the user-defined process graph.
"""
predefined = list(
map(
lambda rec: rec[0],
self.db.query(ProcessGraph.id_openeo).filter_by(
process_definition=ProcessDefinitionEnum.predefined).all())
)
if process_graph_id in predefined:
return ServiceException(
ProcessesService.name,
400,
user_id,
f"The process_graph_id {process_graph_id} is not allowed, as it corresponds"
f" to a predefined process. Please use another process_graph_id. E.g. "
f"user_{process_graph_id}",
internal=False,
links=[])
LOGGER.debug(
f"ProcessGraphId {process_graph_id} is not a predefined process")
return None
def _back_convert_old_process_graph_ids(self,
process_graph_id: str) -> str:
"""Back-Translate reformated process_graph_ids.
Due to backward compatibility some process_graph_ids (id_openeo) do not match the required regex (in the
database) -> they are converted to match and need to be back converted when used internally.
"""
if process_graph_id.startswith("regex_"):
new = process_graph_id.replace("_", "-")
return new[6:]
return process_graph_id
def get_user_id(self, user: Optional[Dict[str, Any]]) -> Optional[str]:
"""Return the user's id if a user object is given."""
return user["id"] if user and "id" in user else None
class CommandInventoryItem:
name = ITEM_COMMAND
db = DatabaseSession(DeclarativeBase)
dispatch = EventDispatcher()
@event_handler(PRODUCTS_COMMAND, 'product_added')
def add_inventory_item(self, data):
""" This is a demo app, so there are some liberties being taken here:
1. The product being added will be added to a random count of sites' inventory
2. Initial stock numbers are going to be generated randomly
3. All sites are able to inventory an item (this may change)
"""
if isinstance(data, str):
data = json.loads(data)
# get the sites
sites = self.db.query(Site).all()
items = []
for site in random.sample(sites, random.randint(1, len(sites))):
item = InventoryItem(product_id=data['product_id'], site_id=site.id)
version = 1
if data.get('version'):
version = (data.get('version') + 1)
if data.get('id'):
id = data.get('id')
else:
id = self.db.execute(Sequence('inventory_items_id_seq'))
item.id = id
item.version = version
self.db.add(item)
self.db.commit()
items.append(item)
for item_data in [{'id': i.id,
'product_id': i.product_id,
'site_id': i.site_id,
'available_stock': i.available_stock,
'restock_threshold': i.restock_threshold,
'version': i.version,
'max_stock_threshold': i.max_stock_threshold,
'on_reorder': i.on_reorder,
'created_at': i.created_at,
'committed_stock': i.committed_stock,
'updated_at': i.updated_at} for i in items]:
self.dispatch(REPLICATE_EVENT, item_data)
@event_handler(None, 'add_item_stock')
def add_item_stock(self, data):
if isinstance(data, str):
data = json.loads(data)
@event_handler(ORDER_COMMAND, ORDER_STATUS_CHANGED_TO_PAID)
def remove_stock_on_order_paid(self, data):
if isinstance(data, str):
data = json.loads(data)
items = data['order_stock_items']
@event_handler(ORDER_COMMAND, ORDER_STATUS_CHANGED_TO_AWAITING_VERIFICATION)
def verify_order_item_availability(self, data):
"""
This method will verify stock across all sites, it will not reject an order
just because one site does not have the inventory to meet the order's needs,
shipping fulfillment will cover the multiple warehouses shipping
:param data:
:return:
"""
if isinstance(data, str):
data = json.loads(data)
confirmed_order_stock_items = []
for i in data['order_stock_items']:
subqry = self.db.query(InventoryItem.site_id,
func.max(InventoryItem.version).label('maxversion')) \
.filter(InventoryItem.product_id == i['product_id']) \
.group_by(InventoryItem.site_id).subquery('t2')
inventory_items = self.db.query(InventoryItem) \
.filter(InventoryItem.product_id == i['product_id']) \
.join(subqry,
(InventoryItem.site_id == subqry.c.site_id) &
(InventoryItem.version == subqry.c.maxversion)) \
.all()
inventory_items = Enumerable(inventory_items)
inventory_count = inventory_items.sum(lambda x: x.available_stock)
has_stock = inventory_count >= i['units']
confirmed_order_stock_items.append({'product_id': i['product_id'],
'has_stock': has_stock})
rejected = all(item['has_stock'] for item in confirmed_order_stock_items)
if not rejected:
payload = {'order_id': data['order_id'],
'order_stock_items': confirmed_order_stock_items}
self.dispatch('rejected_order_stock', payload)
else:
self.dispatch('confirmed_order_stock', {'order_id': data['order_id']})
class ProcessesGraphService:
"""Management of stored process graphs.
"""
name = "process_graphs"
db = DatabaseSession(Base)
process_service = RpcProxy("processes")
data_service = RpcProxy("data")
validator = Validator()
node_parser = NodeParser()
@rpc
def get(self, user_id: str, process_graph_id: str):
import pdb; pdb.set_trace()
try:
process_graph = self.db.query(ProcessGraph).filter_by(id=process_graph_id).first()
if process_graph is None:
return ServiceException(ProcessesService.name, 400, user_id,
"The process_graph with id '{0}' does not exist.".format(process_graph_id), internal=False,
links=["#tag/Job-Management/paths/~1process_graphs~1{process_graph_id}/get"]).to_dict()
# TODO: Permission (e.g admin)
if process_graph.user_id != user_id:
return ServiceException(ProcessesService.name, 401, user_id,
"You are not allowed to access this resource.", internal=False,
links=["#tag/Job-Management/paths/~1process_graphs~1{process_graph_id}/get"]).to_dict()
return {
"status": "success",
"code": 200,
"data": ProcessGraphFullSchema().dump(process_graph).data
}
except Exception as exp:
return ServiceException(ProcessesService.name, 500, user_id, str(exp)).to_dict()
@rpc
def delete(self, user_id: str, process_graph_id: str):
try:
raise Exception("Not implemented yet!")
except Exception as exp:
return ServiceException(ProcessesService.name, 500, user_id, str(exp),
links=["#tag/Job-Management/paths/~1process_graphs~1{process_graph_id}/delete"]).to_dict()
@rpc
def modify(self, user_id: str, process_graph_id: str, **process_graph_args):
try:
raise Exception("Not implemented yet!")
except Exception as exp:
return ServiceException(ProcessesService.name, 500, user_id, str(exp),
links=["#tag/Job-Management/paths/~1process_graphs~1{process_graph_id}/patch"]).to_dict()
@rpc
def get_all(self, user_id: str):
try:
process_graphs = self.db.query(ProcessGraph).order_by(ProcessGraph.created_at).all()
return {
"status": "success",
"code": 200,
"data": ProcessGraphShortSchema(many=True).dump(process_graphs).data
}
except Exception as exp:
return ServiceException(ProcessesService.name, 500, user_id, str(exp),
links=["#tag/Job-Management/paths/~1process_graphs/get"]).to_dict()
@rpc
def create(self, user_id: str, **process_graph_args):
"""The request will ask the back-end to create a new process using the description send in the request body.
Keyword Arguments:
user_id {str} -- The identifier of the user (default: {None})
"""
try:
process_graph_json = deepcopy(process_graph_args.get("process_graph", {}))
validate = self.validate(user_id, process_graph_json)
if validate["status"] == "error":
return validate
# Get all processes
process_response = self.process_service.get_all()
if process_response["status"] == "error":
return process_response
processes = process_response["data"]
process_graph = ProcessGraph(**{"user_id": user_id, **process_graph_args})
nodes = self.node_parser.parse_process_graph(process_graph_json, processes)
imagery_id = None
for idx, node in enumerate(nodes):
process_node = ProcessNode(
user_id=user_id,
seq_num=len(nodes) - (idx + 1),
imagery_id=imagery_id,
process_graph_id=process_graph.id,
**node)
self.db.add(process_node)
imagery_id = process_node.id
process_graph_id = str(process_graph.id)
self.db.add(process_graph)
self.db.commit()
return {
"status": "success",
"code": 201,
"headers": {"Location": "https://openeo.eodc.eu/api/v0/TBD"},
"service_data": process_graph_id
}
except Exception as exp:
return ServiceException(ProcessesService.name, 500, user_id, str(exp)).to_dict()
@rpc
def validate(self, user_id: str, process_graph: dict):
"""The request will ask the back-end to create a new process using the description send in the request body.
Keyword Arguments:
user_id {str} -- The identifier of the user (default: {None})
"""
# TODO: RESPONSE HEADERS -> OpenEO-Costs
try:
# Get all processes
process_response = self.process_service.get_all()
if process_response["status"] == "error":
return process_response
processes = process_response["data"]
# Get all products
product_response = self.data_service.get_all_products()
if product_response["status"] == "error":
return product_response
products = product_response["data"]
self.validator.update_datasets(processes, products)
self.validator.validate_node(process_graph)
return {
"status": "success",
"code": 204
}
except ValidationError as exp:
return ServiceException(ProcessesService.name, 400, user_id, exp.message, internal=False,
links=["#tag/EO-Data-Discovery/paths/~1process_graph/post"]).to_dict()
except Exception as exp:
return ServiceException(ProcessesService.name, 500, user_id, str(exp)).to_dict()
@rpc
def get_nodes(self, user_id: str, process_graph_id: str):
try:
process_graph = self.db.query(ProcessGraph).filter_by(id=process_graph_id).first()
nodes = sorted(process_graph.nodes, key=lambda n: n.seq_num)
return {
"status": "success",
"data": ProcessNodeSchema(many=True).dump(nodes).data
}
except Exception as exp:
return ServiceException(ProcessesService.name, 500, user_id, str(exp)).to_dict()
class JobService:
"""Management of batch processing tasks (jobs) and their results.
"""
name = service_name
db = DatabaseSession(Base)
process_graphs_service = RpcProxy("process_graphs")
data_service = RpcProxy("data")
api_connector = APIConnector()
template_controller = TemplateController()
@rpc
def get(self, user_id: str, job_id: str):
try:
job = self.db.query(Job).filter_by(id=job_id).first()
valid, response = self.authorize(user_id, job_id, job)
if not valid:
return response
response = self.process_graphs_service.get(user_id,
job.process_graph_id)
if response["status"] == "error":
return response
job.process_graph = response["data"]["process_graph"]
return {
"status": "success",
"code": 200,
"data": JobSchemaFull().dump(job).data
}
except Exception as exp:
return ServiceException(
500,
user_id,
str(exp),
links=["#tag/Job-Management/paths/~1jobs~1{job_id}/get"
]).to_dict()
@rpc
def modify(self,
user_id: str,
job_id: str,
process_graph: dict,
title: str = None,
description: str = None,
output: dict = None,
plan: str = None,
budget: int = None):
try:
raise Exception("Not implemented yet!")
except Exception as exp:
return ServiceException(
500,
user_id,
str(exp),
links=["#tag/Job-Management/paths/~1jobs~1{job_id}/patch"
]).to_dict()
@rpc
def delete(self, user_id: str, job_id: str):
try:
raise Exception("Not implemented yet!")
except Exception as exp:
return ServiceException(
500,
user_id,
str(exp),
links=["#tag/Job-Management/paths/~1jobs~1{job_id}/delete"
]).to_dict()
@rpc
def get_all(self, user_id: str):
try:
jobs = self.db.query(Job).filter_by(user_id=user_id).order_by(
Job.created_at).all()
return {
"status": "success",
"code": 200,
"data": JobSchema(many=True).dump(jobs).data
}
except Exception as exp:
return ServiceException(
500,
user_id,
str(exp),
links=["#tag/Job-Management/paths/~1jobs/get"]).to_dict()
@rpc
def create(self,
user_id: str,
process_graph: dict,
title: str = None,
description: str = None,
output: dict = None,
plan: str = None,
budget: int = None):
try:
process_response = self.process_graphs_service.create(
user_id=user_id, **{"process_graph": process_graph})
if process_response["status"] == "error":
return process_response
process_graph_id = process_response["service_data"]
job = Job(user_id, process_graph_id, title, description, output,
plan, budget)
job_id = str(job.id)
self.db.add(job)
self.db.commit()
return {
"status": "success",
"code": 201,
"data": "jobs/" + job_id,
"headers": {
"Location": "jobs/" + job_id
}
}
except Exception as exp:
return ServiceException(
500,
user_id,
str(exp),
links=["#tag/Job-Management/paths/~1jobs/post"]).to_dict()
@rpc
def process(self, user_id: str, job_id: str):
try:
job = self.db.query(Job).filter_by(id=job_id).first()
valid, response = self.authorize(user_id, job_id, job)
if not valid:
raise Exception(response)
job.status = "running"
self.db.commit()
# Get process nodes
response = self.process_graphs_service.get_nodes(
user_id=user_id, process_graph_id=job.process_graph_id)
if response["status"] == "error":
raise Exception(response)
process_nodes = response["data"]
# Get file_paths
filter_args = process_nodes[0]["args"]
response = self.data_service.get_records(
detail="file_path",
user_id=user_id,
name=filter_args["name"],
spatial_extent=filter_args["extent"],
temporal_extent=filter_args["time"])
if response["status"] == "error":
raise Exception(response)
filter_args["file_paths"] = response["data"]
# TODO: Calculate storage size and get storage class
# TODO: Implement Ressource Management
storage_class = "storage-write"
storage_size = "5Gi"
if "local_registry" in environ:
image_registry = environ.get("local_registry")
else:
image_registry = "docker-registry.default.svc:5000"
processing_container = image_registry + "/execution-environment/openeo-processing"
min_cpu = "500m"
max_cpu = "4"
min_ram = "256Mi"
max_ram = "5Gi"
# Create OpenShift objects
pvc = self.template_controller.create_pvc(self.api_connector,
"pvc-" + job.id,
storage_class,
storage_size)
config_map = self.template_controller.create_config(
self.api_connector, "cm-" + job.id, process_nodes)
# Deploy container
logs, metrics = self.template_controller.deploy(
self.api_connector, job.id, processing_container, config_map,
pvc, min_cpu, max_cpu, min_ram, max_ram)
pvc.delete(self.api_connector)
job.logs = logs
job.metrics = metrics
job.status = "finished"
self.db.commit()
return
except Exception as exp:
job.status = job.status + " error: " + exp.__str__(
) #+ ' ' + filter_args
self.db.commit()
return
@rpc
def process_sync(self,
user_id: str,
process_graph: dict,
output: dict = None,
plan: str = None,
budget: int = None):
"""
Creates a processes a job directly using two other functions of this class.
"""
response = self.create(user_id=user_id,
process_graph=process_graph,
output=output,
plan=plan,
budget=budget)
job_id = response['headers']['Location'].split('jobs/')[-1]
_ = self.process(user_id=user_id, job_id=job_id)
return {
"status": "success",
"code": 201,
"headers": {
"Location": "jobs/" + job_id
}
}
@rpc
def estimate(self, user_id: str, job_id: str):
"""
Basic function to return default information about processng costs on back-end.
"""
default_out = {
"costs": 0,
"duration": "null",
"download_included": True,
"expires": "null"
}
return {"status": "success", "code": 200, "data": default_out}
# TODO: If build should be automated using an endpoint e.g. /build the following can be
# activated and adapted
# @rpc
# def build(self, user_id: str):
# try:
# status, log, obj_image_stream = self.template_controller.build(
# self.api_connector,
# environ["CONTAINER_NAME"],
# environ["CONTAINER_TAG"],
# environ["CONTAINER_GIT_URI"],
# environ["CONTAINER_GIT_REF"],
# environ["CONTAINER_GIT_DIR"])
# except Exception as exp:
# return ServiceException(500, user_id, str(exp),
# links=["#tag/Job-Management/paths/~1jobs~1{job_id}~1results/delete"]).to_dict()
@rpc
def cancel_processing(self, user_id: str, job_id: str):
try:
raise Exception("Not implemented yet!")
except Exception as exp:
return ServiceException(
500,
user_id,
str(exp),
links=[
"#tag/Job-Management/paths/~1jobs~1{job_id}~1results/delete"
]).to_dict()
@rpc
def get_results(self, user_id: str, job_id: str):
try:
raise Exception("Not implemented yet!")
except Exception as exp:
return ServiceException(
500,
user_id,
str(exp),
links=[
"#tag/Job-Management/paths/~1jobs~1{job_id}~1results/get"
]).to_dict()
def authorize(self, user_id, job_id, job):
if job is None:
return False, ServiceException(
400,
user_id,
"The job with id '{0}' does not exist.".format(job_id),
internal=False,
links=["#tag/Job-Management/paths/~1data/get"]).to_dict()
# TODO: Permission (e.g admin)
if job.user_id != user_id:
return False, ServiceException(
401,
user_id,
"You are not allowed to access this ressource.",
internal=False,
links=["#tag/Job-Management/paths/~1data/get"]).to_dict()
return True, None
class PlayerService(BaseService):
"""Provides a Nameko service to handle game player activities.
This service does not have any external service dependencies.
Attributes:
name (str): Service name for Nameko
model (cls): Primary model for this particular service
"""
name = "player"
model = PlayerModel
schema = PlayerSchema
session = DatabaseSession(Base)
@rpc
def create(self, name):
"""Creates model objects.
Args:
name (int): The name of the player.
Returns:
dict: created model as dictionary via marshmallow schema.
"""
model = self.create_model(name=name)
return self.schema().dump(model).data
@rpc
def get(self, id):
"""Creates model objects.
Args:
id (int): The primary key of the fetched model.
Returns:
dict: model dictionary if exists, False otherwise.
"""
return self.schema().dump(self.get_model(id)).data
@rpc
def filter(self, **kwargs):
"""Filters model objects by keys and serializes the result.
Args:
id (int): The primary key of the fetched model.
Returns:
dict: list of model dictionaries if exists, False otherwise.
"""
# TODO: error handling
return self.schema(many=True).dump(self.filter_model(**kwargs)).data
@rpc
def update(self, raw_obj, many=False):
"""Updates model objects with keys and serializes the result.
Args:
id (int): The primary key of the fetched model.
Returns:
dict: list of model dictionaries if exists, False otherwise.
"""
# TODO: catch and handle malformed schema errors.
model = self.schema(many=many).load(raw_obj).data
return self.schema().dump(self.update_model(model)).data
class OrdersService(object):
name = 'orders'
products_rpc = RpcProxy('products')
event_dispatcher = EventDispatcher()
order_db = DatabaseSession(DeclarativeBase)
order_detail_db = DatabaseSession(DeclarativeOrderDetail)
hashids = Hashids(min_length=10,
alphabet='0123456789abcdef',
salt='A secret key')
def __init__(self):
self.logger = init_logger(name=self.name)
@rpc
def getOrder(self, hashed_order_id):
order_id = self.hashids.decrypt(hashid=hashed_order_id)
# self.logger.info("order_id: %s" % order_id)
order = self.order_db.query(Order).get(order_id)
if not order:
raise NotFound(
'Order with id {} not found'.format(hashed_order_id))
# Hide real Id from DB
data = OrderSchema().dump(order).data
data['id'] = hashed_order_id
return data
@rpc
def createOrder(self, order_details):
order = Order(order_details=[
OrderDetail(product_id=order_detail['product_id'],
product_name=order_detail['product_name'],
price=order_detail['price'],
quantity=order_detail['quantity'],
currency=order_detail['currency'])
for order_detail in order_details
])
self.order_db.add(order)
self.order_db.commit()
order = OrderSchema().dump(order).data
try:
# self.event_dispatcher('order_created', {
# 'order': order
# })
self.products_rpc.handleOrderCreated({'order': order})
except RemoteError as ex:
self.logger.info("orders.createOrder: %s" % ex)
raise OutOfStock(str(ex))
# self.logger.info("createOrder: order: %s" % order)
# Hide real Id from DB
order['hash_id'] = self.hashids.encrypt(order['id'])
return order
@rpc
def deleteOrder(self, order_id):
order = self.order_db.query(Order).get(order_id)
self.order_db.delete(order)
self.order_db.commit()
@rpc
def showCart(self, customer_id):
try:
# Search order by customer ID and status, to get order id
order = self.order_db.query(Order).filter_by(
customer_id=customer_id, status='CREATED').first()
order_id = order.id
product_list = self.order_detail_db.query(OrderDetail).filter_by(
order_id=order_id).all()
data_list = []
for product in product_list:
data = OrderDetailSchema().dump(product).data
data['id'] = self.hashids.encrypt(product.id)
data_list.append(data)
except Exception as ex:
self.logger.info("orders.updateOrder: %s" % ex)
raise NotFound(
'Order with customer id {} not found'.format(customer_id))
return data_list
@rpc
def processCart(self, order, customer_id):
existed_order = self.order_db.query(Order).filter_by(
status='CREATED', customer_id=customer_id).first()
try:
self.products_rpc.handleCartUpdated({'order': order})
if existed_order:
return self._updateQuantityCart(order, existed_order)
else:
return self._createCart(order, customer_id)
except RemoteError as ex:
self.logger.info("orders.updateOrder: %s" % ex)
raise OutOfStock(str(ex))
except Exception as ex:
self.logger.info("orders.updateOrder: %s" % ex)
raise OutOfStock(str(ex))
def _updateQuantityCart(self, order, existed_order):
temp_dict_1 = {}
temp_dict_2 = {}
for update_product in order['order_details']:
temp_dict_1[update_product['product_id']] = update_product
for existed_product in existed_order.order_details:
temp_dict_2[existed_product.product_id] = {
'product_id': existed_product.product_id,
'order_id': existed_order.id,
'product_name': existed_product.product_name,
'price': existed_product.price,
'quantity': existed_product.quantity
}
temp_dict_2.update(temp_dict_1)
self.order_detail_db.query(OrderDetail).filter_by(
order_id=existed_order.id).delete()
self.order_detail_db.add_all([
OrderDetail(product_id=update_product['product_id'],
order_id=existed_order.id,
product_name=update_product['product_name'],
price=update_product['price'],
quantity=update_product['quantity'],
currency=update_product['currency'])
for update_product in temp_dict_2.values()
])
# Update quantity
self.order_db.commit()
# Quantity = 0
self.order_detail_db.query(OrderDetail).filter_by(quantity=0).delete()
self.order_detail_db.commit()
return "Update successful!"
def _createCart(self, order_data, customer_id):
order = Order(customer_id=customer_id,
order_details=[
OrderDetail(
product_id=order_detail['product_id'],
product_name=order_detail['product_name'],
price=order_detail['price'],
quantity=order_detail['quantity'],
currency=order_detail['currency'],
) for order_detail in order_data["order_details"]
])
self.order_db.add(order)
self.order_db.commit()
order = OrderSchema().dump(order).data
# Hide real Id from DB
order['hash_id'] = self.hashids.encrypt(order['id'])
return "Create successful!"
@rpc
def deleteCart(self, customer_id):
try:
# Search order by customer ID and status
order = self.order_db.query(Order).filter_by(
customer_id=customer_id, status='CREATED').first()
order_id = order.id
logging.warning(">> ", order_id)
# Delete order details
self.order_detail_db.query(OrderDetail).filter_by(
order_id=order_id).delete()
self.order_detail_db.commit()
# Delete order
self.order_db.query(Order).filter_by(customer_id=customer_id,
status='CREATED').delete()
self.order_db.commit()
except Exception as ex:
self.logger.info("orders.updateOrder: %s" % ex)
raise OutOfStock(str(ex))
return "Delete Successful"
@rpc
def checkoutCart(self, customer_id):
try:
order = self.order_db.query(Order).filter_by(
customer_id=customer_id, status='CREATED').first()
order.status = "DONE"
self.order_db.commit()
except Exception as ex:
self.logger.info("orders.updateOrder: %s" % ex)
raise OutOfStock(str(ex))
return "Checkout Successful"
class GameService(BaseService):
"""Provides a Nameko service to handle card game activities.
This service has external service dependencies. It depends on
1. PlayerService
2. DeckService
Attributes:
name (str): Service name for Nameko
model (cls): Primary model for this particular service
schema (cls): Main marshmallow schema class for this service
session (DataseSession): SQLAlchemy session
player_rpc (RpcProxy(PlayerService)): users / players
deck_rpc (RpcProxy(DeckService)): deck of cards
player_cards (int): no. of cards for player at start
dealer_cards (int): no. of visible cards for dealer at start
dealer_cards_hidden (int): no. hidden cards for dealer at start
"""
name = "game"
model = GameModel
schema = GameSchema
session = DatabaseSession(Base)
# we depend on these RPC interface.
player_rpc = RpcProxy("player")
deck_rpc = RpcProxy("deck")
# TODO: move these variables into the config file using plugin.
player_cards = 2
dealer_cards = 1
dealer_cards_hidden = 1
def _setup_game(self, model):
if not (self.dealer_cards + self.dealer_cards_hidden) == 2:
# TODO: don't use generic exception, use ValueError or better.
raise Exception('The dealer must have exactly two cards.')
if not (self.player_cards == 2):
# TODO: don't use generic exception, use ValueError or better.
raise Exception('The player must have exactly two cards.')
# When the game starts, the player is given 2 random cards
for i in range(1, self.player_cards + 1):
self._hit(model)
model.status = 1
self.update_model(model)
# The dealer is given 0 or more card(s) that are hidden.
for i in range(1, self.dealer_cards + 1):
self._hit(model, hidden=False)
# The dealer is given 0 or more card(s) that are visible.
for i in range(1, self.dealer_cards_hidden + 1):
self._hit(model, hidden=True)
model.status = 10
self.update_model(model)
def _get_score(self, hand):
score = 0
for card in hand:
points = int(card[1:])
if points > 10:
points = 10
score += points
return score
def _check(self, model):
# TODO: refactor this into check, check_player and check_dealer
# We are checking for a player
if model.status == 0 or model.status == 10:
model.player_score = self._get_score(model.player_hand)
# PlayerBlackjack
if model.player_score == 21:
model.status = 21
# PlayerBust
if model.player_score > 21:
model.status = 30
# We are checking for a dealer
elif model.status == 1 or model.status == 11:
model.dealer_score = self._get_score(model.dealer_hand_hidden +
model.dealer_hand)
# TODO: check player_score is set here.
# TODO: check if dealer_score is greater than player_score
# DealerWins
if model.dealer_score == 21:
model.status = 31
# DealerBust
if model.dealer_score > 21:
model.status = 30
else:
raise (ValueError('The game has an unknown status: {}'.format(
model.status)))
return model
def _hit(self, model, hidden=None):
# Hidden - True for dealing hidden dealer card, False for shown dealer
# card, None for player card
deck_id = model.deck_id
# TODO: check if we should be dealing a card or this one will
# get thrown away after we throw an error.
card = self.deck_rpc.deal_card(deck_id)
# It is the dealers turn.
if model.status == 1 or model.status == 11:
# TODO: be careful, we remove a card from the deck before putting
# it in a hand, we could lose the card if we throw an error.
if hidden:
hand = list(model.dealer_hand_hidden)
hand.append(card)
model.dealer_hand_hidden = hand
else:
hand = list(model.dealer_hand)
hand.append(card)
model.dealer_hand = hand
# It is the players turn.
elif model.status == 0 or model.status == 10:
# TODO: be careful, we remove a card from the deck before putting
# it in a hand, we could lose the card if we throw an error.
hand = list(model.player_hand)
hand.append(card)
model.player_hand = hand
# User has stuck, don't let them fetch any more cards.
elif model.status in [11, 12]:
# We are stuck, we don't need to update anything.
return None
else:
raise NotImplementedError('Implement _hit for status: {}'.format(
model.status))
# Check that model has been saved, if it has we can run _check()
model = self._check(model)
self.update_model(model)
return model
def _toggle_ace(self, model, dealer=False):
hand = None
if dealer:
# iterate through dealer cards hidden
for card in model.dealer_hand_hidden:
if int(card[1:]) == 1:
# we have an ace
# TODO: add code to ensure we can't end up with two aces
# high as that is insta-bust
model.dealer_ace_high_hidden ^= True
hand = list(self.dealer_hand_hidden)
model.dealer_hand = hand
# iterate through dealer cards
for card in model.dealer_hand:
if int(card[1:]) == 1:
# we have an ace
# TODO: add code to ensure we can't end up with two aces
# high as that is insta-bust
model.dealer_ace_high ^= True
hand = list(self.dealer_hand)
model.dealer_hand = hand
else:
# iterate through user cards
for card in model.player_hand:
if int(card[1:]) == 1:
# we have an ace
model.player_ace_high = not model.player_ace_high
hand = list(self.player_hand)
model.player_hand = hand
model = self.check(model)
self.update_model(model)
return model
@rpc
def create(self, player_id=None):
"""Creates model objects.
Args:
player_id (int): The foreign key for player.
Returns:
int: primary key of created model
"""
# TODO: check player_id exists.
logging.debug('GameService.create({})'.format(player_id))
player = self.player_rpc.get(player_id)
print(player)
if player:
# create a deck whenever we create a game.
model = self.create_model(
player_id=player['id'],
deck_id=self.deck_rpc.create()['id'],
status=0, # Status: Created.
)
self._setup_game(model)
return self.schema().dump(model).data
else:
# TODO: exception handling, how to pass errors?
raise Exception(
'Player with id={} does not exist'.format(player_id))
@rpc
def get(self, id):
"""Creates model objects.
Args:
id (int): The primary key of the fetched model.
Returns:
GameModel: SQLAlchemy model if exists, False otherwise
"""
return self.schema().dump(self.get_model(id)).data
@rpc
def hit(self, id):
model = self.get_model(id)
if model.status == 10:
return self.schema().dump(self._hit(model)).data
else:
# TODO: check status and return appropriate error message.
return False
@rpc
def toggle_ace(self, id):
model = self.get_model(id)
return self.schema().dump(self._toggle_ace(model)).data
@rpc
def stick(self, id):
# TODO: refactor stick into _stick
model = self.get_model(id)
if model.status == 10:
model.status = 11
self.update_model(model)
while True:
if (model.player_score > 21):
print('PlayerBust')
model.status = 30
break
elif model.dealer_score > 21:
print('DealerBust')
model.status = 20
break
elif model.status == 12:
print('DealerWins')
model.status = 31
break
elif model.dealer_score == 21:
print('DealerStuck')
model.status = 12
elif model.dealer_score > model.player_score:
print('DealerStuck')
model.status = 12
# The player can't win outright as dealer won't stick until
# they win.
print('DealerHit')
self._hit(model)
self.update_model(model)
# reveal the dealers card by setting exclude to blank.
return GameSchemaStuck().dump(model).data
else:
# TODO: check status and return appropriate error message.
return False
class NewsService:
name = 'news_service'
db = DatabaseSession(Base)
@http('POST', '/news')
def create_news(self, request):
schema = NewsSchema(strict=True)
try:
data = schema.loads(request.get_data(as_text=True)).data
except ValueError as error:
return Exception(error)
news = NewsModel(data)
self.db.add(news)
self.db.commit()
respose_data = schema.dump(news).data
return 201, json.dumps(respose_data)
@http('GET', '/news')
def list_news(self, request):
news = self.db.query(NewsModel).all()
respose_data = NewsSchema().dump(news, many=True).data
return 200, json.dumps(respose_data)
@http('GET', '/news/<int:news_id>')
def get_news(self, request, news_id):
news = self.db.query(NewsModel).get(news_id)
if not news:
return Exception('error: news not found')
respose_data = NewsSchema().dump(news).data
return 200, json.dumps(respose_data)
@http('PUT', '/news/<int:news_id>')
def udpate_news(self, request, news_id):
schema = NewsSchema(strict=True)
try:
data = schema.loads(
request.get_data(as_text=True),
partial=True,
).data
except ValueError as error:
return Exception(error)
news = self.db.query(NewsModel).get(news_id)
news.author = data.get('author')
news.title = data.get('title')
news.content = data.get('content')
news.is_active = data.get('is_active')
news.tags = data.get('tags')
self.db.commit()
respose_data = NewsSchema().dump(news).data
return 200, json.dumps(respose_data)
@http('DELETE', '/news/<int:news_id>')
def delete_news(self, request, news_id):
news = self.db.query(NewsModel).get(news_id)
self.db.delete(news)
self.db.commit()
return 204, 'News ID {} Deleted'.format(news_id)