def check_condition_to_start(version_id): global product_api product_api = Products() crawl_api = Crawls() model_api = Models() try: model = model_api.get_model(TEXT_CLASSIFICATION_MODEL_TYPE, version_id=version_id) if model is not None: return False log.info("check_condition_to_start") # Check if crawling process is done total_crawl_size = crawl_api.get_size_crawls(version_id) crawled_size = crawl_api.get_size_crawls(version_id, status='done') if total_crawl_size != crawled_size: return False total_product_size = product_api.get_size_products(version_id) processed_product_size = product_api.get_size_products( version_id, is_processed_for_text_class_model=True) not_processed_product_size = product_api.get_size_products( version_id, is_processed_for_text_class_model=False) if (processed_product_size + not_processed_product_size) == total_product_size: return False except Exception as e: log.error(str(e)) return True
def check_condition_to_start(version_id): global product_api product_api = Products() crawl_api = Crawls() try: log.info("check_condition_to_start") # Check if crawling process is done total_crawl_size = crawl_api.get_size_crawls(version_id) crawled_size = crawl_api.get_size_crawls(version_id, status='done') if total_crawl_size != crawled_size: return False queue_size = rconn.llen(REDIS_PRODUCT_TEXT_MODEL_PROCESS_QUEUE) if queue_size > 0: return False total_product_size = product_api.get_size_products(version_id) processed_product_size = product_api.get_size_products(version_id, is_processed_for_text_class_model=True) not_processed_product_size = product_api.get_size_products(version_id, is_processed_for_text_class_model=False) if (processed_product_size + not_processed_product_size) == total_product_size: return False except Exception as e: log.error(str(e)) return True
def check_condition_to_start(version_id): global product_api product_api = Products() try: # Check Classifying processing process is done total_product_size = product_api.get_size_products(version_id) classified_size = product_api.get_size_products(version_id, is_classified=True) if total_product_size != classified_size: return False # Check Object classifying process is done queue_size = rconn.llen(REDIS_OBJECT_INDEX_QUEUE) if queue_size != 0: return False except Exception as e: log.error(str(e)) return True
def check_condition_to_start(version_id): global product_api product_api = Products() crawl_api = Crawls() try: # Check Object classifying process is done queue_size = rconn.llen(REDIS_PRODUCT_CLASSIFY_QUEUE) if queue_size != 0: return False # Check Crawling process is done total_crawl_size = crawl_api.get_size_crawls(version_id) crawled_size = crawl_api.get_size_crawls(version_id, status='done') if total_crawl_size != crawled_size: return False # Check if all images are processed total_product_size = product_api.get_size_products(version_id) available_product_size = product_api.get_size_products(version_id, is_available=True) unavailable_product_size = product_api.get_size_products(version_id, is_available=False) # processed_size = product_api.get_size_products(version_id, is_processed=True) if (available_product_size + unavailable_product_size) != total_product_size: return False # Check Classifying processing process is done classified_size = product_api.get_size_products(version_id, is_classified=True) not_classified_size = product_api.get_size_products(version_id, is_classified=False) if (classified_size + not_classified_size) == total_product_size: return False except Exception as e: log.error(str(e)) return True
def check_condition_to_start(version_id): global product_api product_api = Products() crawl_api = Crawls() try: log.info("check_condition_to_start") # Check if image processing queue is empty queue_size = rconn.llen(REDIS_PRODUCT_IMAGE_PROCESS_QUEUE) if queue_size != 0: return False # Check if crawling process is done total_crawl_size = crawl_api.get_size_crawls(version_id) crawled_size = crawl_api.get_size_crawls(version_id, status='done') if total_crawl_size != crawled_size: return False # Check if all images are processed total_product_size = product_api.get_size_products(version_id) available_product_size = product_api.get_size_products( version_id, is_available=True) unavailable_product_size = product_api.get_size_products( version_id, is_available=False) # processed_size = product_api.get_size_products(version_id, is_processed=True) if (available_product_size + unavailable_product_size) == total_product_size: return False except Exception as e: log.error(str(e)) return True
from __future__ import print_function import time import stylelens_product from stylelens_product.products import Products from pprint import pprint # create an instance of the API class api_instance = Products() version_id = "5a4dacce4dfd7d90b8837719" try: api_response = api_instance.get_size_products(version_id=version_id) pprint('total products: ' + str(api_response)) api_response = api_instance.get_size_products(version_id=version_id, is_processed=True) pprint('Processed products: ' + str(api_response)) api_response = api_instance.get_size_products(version_id=version_id, is_classified=True) pprint('Classified products: ' + str(api_response)) except Exception as e: print("Exception when calling get_size_products: %s\n" % e)