def main(args):
# Initialize Trace Context
global trace_context
trace_context = TraceContext(
trace_id=args.get('__OW_TRACE_ID',
os.environ.get('__OW_TRACE_ID', generate_id())),
service_name='fetchProductImage',
transaction_id=os.environ.get('__OW_TRANSACTION_ID', ''),
tracer_endpoint='192.168.178.62',
parent_id=args.get('__PARENT_TRACE_ID', ''),
action_trace_id=os.environ.get('__OW_TRACE_ID', ''))
# Initialize parameters
image_urls = args.get("imageUrls", [])
shop_key = args.get("shopKey", "771d87188d568ddd")
with Span(span_name='fetch_images',
trace_context=trace_context) as wrapper_context:
thumbnails = []
for image_url in image_urls:
with Span(span_name='fetch_image',
trace_context=trace_context,
parent_id=wrapper_context.id) as wrapper_image_context:
global _WRAPPER_PARENT_SPAN_ID_
_WRAPPER_PARENT_SPAN_ID_ = wrapper_image_context.id
image_file = fetch_image_from_url(image_url.get("imageUrl"))
filename = save_file_in_minio(
image_file, shop_key, image_url.get("externalProductId"),
image_url.get("order"))
if int(image_url.get("order")) == 0:
thumbnails.append(filename)
call_thumbnail_generator(thumbnails)
return {"message": "hi"}
def save_file_in_minio(img_file, shop_key, product_id, order):
"""
creates a file in the minio stores 'productstore' bucket
:param order: The order this image belongs to in a later preview
:param product_id: The external product id
:param shop_key: A string that is used to specify a shop
:param img_file: A object that is an instance of File
:return:
"""
with Span(span_name='save_file_in_minio',
trace_context=trace_context,
parent_id=_WRAPPER_PARENT_SPAN_ID_):
filename = "{}:{}:{}".format(order, product_id, shop_key)
minio_client = Minio('{}'.format(_INTERNAL_DATA_STORE_ENDPOINT_),
access_key=_MINIO_ACCESS_KEY_,
secret_key=_MINIO_SECRET_KEY_,
secure=False)
try:
with io.BytesIO(img_file.file) as file:
minio_client.put_object('productimages', filename, file,
img_file.size)
print("published file: {}".format(filename))
return filename
except ResponseError as err:
print(err)
return "ERROR"
def main(args):
try:
# Initialize trace context
global trace_context
trace_context = TraceContext(
trace_id=args.get('__OW_TRACE_ID',
os.environ.get('__OW_TRACE_ID', generate_id())),
service_name='unifyFormat',
transaction_id=os.environ.get('__OW_TRANSACTION_ID', ''),
tracer_endpoint=_TRACER_ENDPOINT_,
parent_id=args.get('__PARENT_TRACE_ID', ''),
action_trace_id=os.environ.get('__OW_TRACE_ID', ''))
# initialize parameters
filename = str(args.get("filename"))
shop_key = str(args.get("shopKey"))
csv_file = get_csv_file(filename)
products = process_products(csv_file.csv_lines(), shop_key)
# TODO: Logic for failures (check status)
with Span(span_name='invoke_productsApi', trace_context=trace_context):
invoke_action("productsApi",
os.environ.get('__OW_API_HOST',
_OPENWHISK_HOST_ENDPOINT_),
os.environ.get('__OW_API_KEY', _OPENWHISK_KEY_),
data={
'__OW_TRACE_ID': trace_context.trace_id,
'products': products
},
ignore_certs=True)
return {
'__OW_TRACE_ID': trace_context.trace_id,
'products': products.get('products', [])
}
except Exception as e:
return {"error": "{}".format(e)}
def save_file_in_minio(csv_file, shop_key):
"""
creates a file in the minio stores 'productstore' bucket
:param shop_key: A string that is used to specify a shop
:param csv_file: A object that is an instance of CsvFile
:return:
"""
with Span(span_name='save_file_in_minio', trace_context=trace_context):
filename = "{}:{}".format(generate_id(), shop_key)
minio_client = Minio('{}'.format(_INTERNAL_DATA_STORE_ENDPOINT_),
access_key=_MINIO_ACCESS_KEY_,
secret_key=_MINIO_SECRET_KEY_,
secure=False)
try:
with io.BytesIO(csv_file.file) as file:
minio_client.put_object('productstore',
filename,
file,
csv_file.size,
content_type='application/csv')
print("published file: {}".format(filename))
return filename
except ResponseError as err:
print(err)
return "ERROR"
def main(args):
trace_context = TraceContext(
trace_id=args.get('__OW_TRACE_ID',
os.environ.get('__OW_TRACE_ID', generate_id())),
service_name='productsApi',
transaction_id=os.environ.get('__OW_TRANSACTION_ID', ''),
tracer_endpoint=_TRACER_ENDPOINT_,
parent_id=args.get('__PARENT_TRACE_ID', ''),
action_trace_id=os.environ.get('__OW_TRACE_ID', ''))
with Span(span_name='publish_products', trace_context=trace_context):
result = {'message': 'Hello World!'}
return result
def fetch_image_from_url(image_url):
"""
fetches a image and returns it in bytes
:param image_url: A url to an image
:return: image data in bytes
"""
with Span(span_name='fetch_image_from_url',
trace_context=trace_context,
parent_id=_WRAPPER_PARENT_SPAN_ID_):
img_data = requests.get(image_url).content
with io.BytesIO(img_data) as f:
img_file = File(file=f.read(), size=f.getbuffer().nbytes)
return img_file
def call_fetch_product_image_in_batches(csv_lines, max_batch_amount, shop_key):
"""
Processes csv lines and extracts instances of the Class ImageUrl to a list of dicts.
Passes this list to asynchronously called fetchProductImages actions
:param csv_lines: A list of strings that represent all the lines of the handled csv file
:param max_batch_amount: The maximum amount of batches that should be handled by asynchronously called
fetchProductImages actions
:param shop_key: A string that represent a shop
:return:
"""
with Span(trace_context=trace_context,
span_name='call_fetch_product_image_in_batches'):
first_line = csv_lines.pop(0)
image_urls_index = first_line.split(',').index('IMAGEURLS')
id_index = first_line.split(',').index('ID')
image_urls = []
# create ImageUrl instances
for line in csv_lines:
product_id = line.split(',')[id_index]
extraced_image_url_list = line.split(',')[image_urls_index].split(
'|')
count = 0
for image_url in extraced_image_url_list:
image_urls.append(
ImageUrl(image_url, product_id, count).to_dict())
count += 1
# create batches
max_batch_size = int(len(image_urls) / max_batch_amount) + 1
batches = [
image_urls[x:x + max_batch_size]
for x in range(0, len(image_urls), max_batch_size)
]
# call fetchProductImages
for x in range(0, max_batch_amount):
print(batches[x])
print("activationId:{}".format(
invoke_action_async('fetchProductImages',
os.environ.get('__OW_API_HOST',
_OPENWHISK_HOST_ENDPOINT_),
os.environ.get('__OW_API_KEY',
_OPENWHISK_KEY_),
data={
'__OW_TRACE_ID':
trace_context.trace_id,
'imageUrls': batches[x],
'shopKey': shop_key
},
ignore_certs=True)))
def get_csv_file(filename):
"""
gets a csv file from a given url and returns it
:return: returns a csv file as instance of CsvFile
"""
with Span(span_name='fetch_csv_file', trace_context=trace_context):
filename = filename
bucket = "productstore"
endpoint = _INTERNAL_DATA_STORE_ENDPOINT_
minio_client = Minio('{}'.format(endpoint),
access_key=_MINIO_ACCESS_KEY_,
secret_key=_MINIO_SECRET_KEY_,
secure=False)
file = minio_client.get_object(bucket, filename)
with io.BytesIO(file.read()) as f:
csv_file = CsvFile(file=f.read(), size=f.getbuffer().nbytes)
return csv_file
def fetch_csv_file(csv_url):
"""
fetches a csv file from a given url and returns it
:param csv_url: a url to an file hosted on a minio server
:return: returns a csv file as instance of CsvFile
"""
splitted_csv_url = csv_url.split("/")
filename = splitted_csv_url[-1]
bucket = splitted_csv_url[-2]
endpoint = splitted_csv_url[-3]
with Span(span_name='fetch_csv_file', trace_context=trace_context):
minio_client = Minio('{}'.format(endpoint),
access_key=_MINIO_ACCESS_KEY_,
secret_key=_MINIO_SECRET_KEY_,
secure=False)
file = minio_client.get_object(bucket, filename)
with io.BytesIO(file.read()) as f:
csv_file = CsvFile(file=f.read(), size=f.getbuffer().nbytes)
return csv_file
def call_thumbnail_generator(filenames):
"""
Calls the thumbnail generator for batches of filenames that qualify as thumbnail
:param filenames: a list of filenames inside the productimages bucket
:return: void
"""
with Span(span_name='call_thumbnail_generator',
trace_context=trace_context,
parent_id=_WRAPPER_PARENT_SPAN_ID_,
message=Message(key="Filenames", value=filenames)) as parent:
invoke_action('thumbnailGenerator',
os.environ.get('__OW_API_HOST',
_OPENWHISK_HOST_ENDPOINT_),
os.environ.get('__OW_API_KEY', _OPENWHISK_KEY_),
data={
'__OW_TRACE_ID': trace_context.trace_id,
'__PARENT_TRACE_ID': parent.id,
'imageNames': filenames
},
ignore_certs=True)
def process_products(csv_lines, shop_key):
"""
Processed products from csv files to a internal presentation as dicts
:param shop_key: A string that represent a shop
:param csv_lines: A list of strings that represent the lines in a csv file
:return: a status that tells if the extraction of products failed or not, and if not, it contains also the products
in a list of dicts
"""
with Span(span_name='process_products', trace_context=trace_context):
first_line = csv_lines.pop(0).split(',')
products = []
# get indexes of all fields
try:
product_name_index = first_line.index("PNAME")
product_description_index = first_line.index("PDESCRIPTION")
product_price_index = first_line.index("PPRICE")
delivery_time_index = first_line.index("DTIME")
delivery_text_index = first_line.index("DTEXT")
quantity_index = first_line.index("QUANTITY")
id_index = first_line.index("ID")
image_urls_index = first_line.index("IMAGEURLS")
except:
print('could not read relevant field')
return {'status': 'failed'}
# process lines
for line in csv_lines:
line = line.split(',')
image_url_count = get_image_count(line[image_urls_index])
product_id = line[id_index]
quantity = line[quantity_index]
delivery_text = line[delivery_text_index]
delivery_time = line[delivery_time_index]
product_price = line[product_price_index]
product_description = line[product_description_index]
product_name = line[product_name_index]
# check if values are mappable to the expected datatypes
try:
int(product_id)
except:
print('id is not an integer skipping line')
continue
try:
int(quantity)
except:
print('quantity is not an integer skipping line')
continue
try:
int(quantity)
except:
print('quantity is not an integer skipping line')
continue
try:
str(delivery_text)
except:
print(
'delivery text is not mappable to a string skipping line')
continue
try:
int(delivery_time)
except:
print('delivery time is not an integer skipping line')
continue
try:
int(product_price)
except:
print('product price is not an integer skipping line')
continue
try:
str(product_description)
except:
print(
'product description is not mappable to a string skipping line'
)
continue
try:
str(product_name)
except:
print('product name is not mappable to a string skipping line')
continue
# create image filenames
images = []
for i in range(0, image_url_count):
images.append("{}:{}:{}".format(i, product_id, shop_key))
# The thumbnail would contain another value if the thumbnail generator would be fully implemented
thumbnail = "{}:{}:{}".format(0, product_id, shop_key)
products.append(
Product(thumbnail=thumbnail,
name=product_name,
product_images=images,
external_id=product_id,
quantity=quantity,
description=product_description,
delivery_time=delivery_time,
delivery_text=delivery_text,
price=product_price,
shop_key=shop_key).to_dict())
return {"status": "success", "products": products}