def main(args): # Initialize Trace Context global trace_context trace_context = TraceContext( trace_id=args.get('__OW_TRACE_ID', os.environ.get('__OW_TRACE_ID', generate_id())), service_name='fetchProductImage', transaction_id=os.environ.get('__OW_TRANSACTION_ID', ''), tracer_endpoint='192.168.178.62', parent_id=args.get('__PARENT_TRACE_ID', ''), action_trace_id=os.environ.get('__OW_TRACE_ID', '')) # Initialize parameters image_urls = args.get("imageUrls", []) shop_key = args.get("shopKey", "771d87188d568ddd") with Span(span_name='fetch_images', trace_context=trace_context) as wrapper_context: thumbnails = [] for image_url in image_urls: with Span(span_name='fetch_image', trace_context=trace_context, parent_id=wrapper_context.id) as wrapper_image_context: global _WRAPPER_PARENT_SPAN_ID_ _WRAPPER_PARENT_SPAN_ID_ = wrapper_image_context.id image_file = fetch_image_from_url(image_url.get("imageUrl")) filename = save_file_in_minio( image_file, shop_key, image_url.get("externalProductId"), image_url.get("order")) if int(image_url.get("order")) == 0: thumbnails.append(filename) call_thumbnail_generator(thumbnails) return {"message": "hi"}
def save_file_in_minio(img_file, shop_key, product_id, order): """ creates a file in the minio stores 'productstore' bucket :param order: The order this image belongs to in a later preview :param product_id: The external product id :param shop_key: A string that is used to specify a shop :param img_file: A object that is an instance of File :return: """ with Span(span_name='save_file_in_minio', trace_context=trace_context, parent_id=_WRAPPER_PARENT_SPAN_ID_): filename = "{}:{}:{}".format(order, product_id, shop_key) minio_client = Minio('{}'.format(_INTERNAL_DATA_STORE_ENDPOINT_), access_key=_MINIO_ACCESS_KEY_, secret_key=_MINIO_SECRET_KEY_, secure=False) try: with io.BytesIO(img_file.file) as file: minio_client.put_object('productimages', filename, file, img_file.size) print("published file: {}".format(filename)) return filename except ResponseError as err: print(err) return "ERROR"
def main(args): try: # Initialize trace context global trace_context trace_context = TraceContext( trace_id=args.get('__OW_TRACE_ID', os.environ.get('__OW_TRACE_ID', generate_id())), service_name='unifyFormat', transaction_id=os.environ.get('__OW_TRANSACTION_ID', ''), tracer_endpoint=_TRACER_ENDPOINT_, parent_id=args.get('__PARENT_TRACE_ID', ''), action_trace_id=os.environ.get('__OW_TRACE_ID', '')) # initialize parameters filename = str(args.get("filename")) shop_key = str(args.get("shopKey")) csv_file = get_csv_file(filename) products = process_products(csv_file.csv_lines(), shop_key) # TODO: Logic for failures (check status) with Span(span_name='invoke_productsApi', trace_context=trace_context): invoke_action("productsApi", os.environ.get('__OW_API_HOST', _OPENWHISK_HOST_ENDPOINT_), os.environ.get('__OW_API_KEY', _OPENWHISK_KEY_), data={ '__OW_TRACE_ID': trace_context.trace_id, 'products': products }, ignore_certs=True) return { '__OW_TRACE_ID': trace_context.trace_id, 'products': products.get('products', []) } except Exception as e: return {"error": "{}".format(e)}
def save_file_in_minio(csv_file, shop_key): """ creates a file in the minio stores 'productstore' bucket :param shop_key: A string that is used to specify a shop :param csv_file: A object that is an instance of CsvFile :return: """ with Span(span_name='save_file_in_minio', trace_context=trace_context): filename = "{}:{}".format(generate_id(), shop_key) minio_client = Minio('{}'.format(_INTERNAL_DATA_STORE_ENDPOINT_), access_key=_MINIO_ACCESS_KEY_, secret_key=_MINIO_SECRET_KEY_, secure=False) try: with io.BytesIO(csv_file.file) as file: minio_client.put_object('productstore', filename, file, csv_file.size, content_type='application/csv') print("published file: {}".format(filename)) return filename except ResponseError as err: print(err) return "ERROR"
def main(args): trace_context = TraceContext( trace_id=args.get('__OW_TRACE_ID', os.environ.get('__OW_TRACE_ID', generate_id())), service_name='productsApi', transaction_id=os.environ.get('__OW_TRANSACTION_ID', ''), tracer_endpoint=_TRACER_ENDPOINT_, parent_id=args.get('__PARENT_TRACE_ID', ''), action_trace_id=os.environ.get('__OW_TRACE_ID', '')) with Span(span_name='publish_products', trace_context=trace_context): result = {'message': 'Hello World!'} return result
def fetch_image_from_url(image_url): """ fetches a image and returns it in bytes :param image_url: A url to an image :return: image data in bytes """ with Span(span_name='fetch_image_from_url', trace_context=trace_context, parent_id=_WRAPPER_PARENT_SPAN_ID_): img_data = requests.get(image_url).content with io.BytesIO(img_data) as f: img_file = File(file=f.read(), size=f.getbuffer().nbytes) return img_file
def call_fetch_product_image_in_batches(csv_lines, max_batch_amount, shop_key): """ Processes csv lines and extracts instances of the Class ImageUrl to a list of dicts. Passes this list to asynchronously called fetchProductImages actions :param csv_lines: A list of strings that represent all the lines of the handled csv file :param max_batch_amount: The maximum amount of batches that should be handled by asynchronously called fetchProductImages actions :param shop_key: A string that represent a shop :return: """ with Span(trace_context=trace_context, span_name='call_fetch_product_image_in_batches'): first_line = csv_lines.pop(0) image_urls_index = first_line.split(',').index('IMAGEURLS') id_index = first_line.split(',').index('ID') image_urls = [] # create ImageUrl instances for line in csv_lines: product_id = line.split(',')[id_index] extraced_image_url_list = line.split(',')[image_urls_index].split( '|') count = 0 for image_url in extraced_image_url_list: image_urls.append( ImageUrl(image_url, product_id, count).to_dict()) count += 1 # create batches max_batch_size = int(len(image_urls) / max_batch_amount) + 1 batches = [ image_urls[x:x + max_batch_size] for x in range(0, len(image_urls), max_batch_size) ] # call fetchProductImages for x in range(0, max_batch_amount): print(batches[x]) print("activationId:{}".format( invoke_action_async('fetchProductImages', os.environ.get('__OW_API_HOST', _OPENWHISK_HOST_ENDPOINT_), os.environ.get('__OW_API_KEY', _OPENWHISK_KEY_), data={ '__OW_TRACE_ID': trace_context.trace_id, 'imageUrls': batches[x], 'shopKey': shop_key }, ignore_certs=True)))
def get_csv_file(filename): """ gets a csv file from a given url and returns it :return: returns a csv file as instance of CsvFile """ with Span(span_name='fetch_csv_file', trace_context=trace_context): filename = filename bucket = "productstore" endpoint = _INTERNAL_DATA_STORE_ENDPOINT_ minio_client = Minio('{}'.format(endpoint), access_key=_MINIO_ACCESS_KEY_, secret_key=_MINIO_SECRET_KEY_, secure=False) file = minio_client.get_object(bucket, filename) with io.BytesIO(file.read()) as f: csv_file = CsvFile(file=f.read(), size=f.getbuffer().nbytes) return csv_file
def fetch_csv_file(csv_url): """ fetches a csv file from a given url and returns it :param csv_url: a url to an file hosted on a minio server :return: returns a csv file as instance of CsvFile """ splitted_csv_url = csv_url.split("/") filename = splitted_csv_url[-1] bucket = splitted_csv_url[-2] endpoint = splitted_csv_url[-3] with Span(span_name='fetch_csv_file', trace_context=trace_context): minio_client = Minio('{}'.format(endpoint), access_key=_MINIO_ACCESS_KEY_, secret_key=_MINIO_SECRET_KEY_, secure=False) file = minio_client.get_object(bucket, filename) with io.BytesIO(file.read()) as f: csv_file = CsvFile(file=f.read(), size=f.getbuffer().nbytes) return csv_file
def call_thumbnail_generator(filenames): """ Calls the thumbnail generator for batches of filenames that qualify as thumbnail :param filenames: a list of filenames inside the productimages bucket :return: void """ with Span(span_name='call_thumbnail_generator', trace_context=trace_context, parent_id=_WRAPPER_PARENT_SPAN_ID_, message=Message(key="Filenames", value=filenames)) as parent: invoke_action('thumbnailGenerator', os.environ.get('__OW_API_HOST', _OPENWHISK_HOST_ENDPOINT_), os.environ.get('__OW_API_KEY', _OPENWHISK_KEY_), data={ '__OW_TRACE_ID': trace_context.trace_id, '__PARENT_TRACE_ID': parent.id, 'imageNames': filenames }, ignore_certs=True)
def process_products(csv_lines, shop_key): """ Processed products from csv files to a internal presentation as dicts :param shop_key: A string that represent a shop :param csv_lines: A list of strings that represent the lines in a csv file :return: a status that tells if the extraction of products failed or not, and if not, it contains also the products in a list of dicts """ with Span(span_name='process_products', trace_context=trace_context): first_line = csv_lines.pop(0).split(',') products = [] # get indexes of all fields try: product_name_index = first_line.index("PNAME") product_description_index = first_line.index("PDESCRIPTION") product_price_index = first_line.index("PPRICE") delivery_time_index = first_line.index("DTIME") delivery_text_index = first_line.index("DTEXT") quantity_index = first_line.index("QUANTITY") id_index = first_line.index("ID") image_urls_index = first_line.index("IMAGEURLS") except: print('could not read relevant field') return {'status': 'failed'} # process lines for line in csv_lines: line = line.split(',') image_url_count = get_image_count(line[image_urls_index]) product_id = line[id_index] quantity = line[quantity_index] delivery_text = line[delivery_text_index] delivery_time = line[delivery_time_index] product_price = line[product_price_index] product_description = line[product_description_index] product_name = line[product_name_index] # check if values are mappable to the expected datatypes try: int(product_id) except: print('id is not an integer skipping line') continue try: int(quantity) except: print('quantity is not an integer skipping line') continue try: int(quantity) except: print('quantity is not an integer skipping line') continue try: str(delivery_text) except: print( 'delivery text is not mappable to a string skipping line') continue try: int(delivery_time) except: print('delivery time is not an integer skipping line') continue try: int(product_price) except: print('product price is not an integer skipping line') continue try: str(product_description) except: print( 'product description is not mappable to a string skipping line' ) continue try: str(product_name) except: print('product name is not mappable to a string skipping line') continue # create image filenames images = [] for i in range(0, image_url_count): images.append("{}:{}:{}".format(i, product_id, shop_key)) # The thumbnail would contain another value if the thumbnail generator would be fully implemented thumbnail = "{}:{}:{}".format(0, product_id, shop_key) products.append( Product(thumbnail=thumbnail, name=product_name, product_images=images, external_id=product_id, quantity=quantity, description=product_description, delivery_time=delivery_time, delivery_text=delivery_text, price=product_price, shop_key=shop_key).to_dict()) return {"status": "success", "products": products}