def merge_computation(self):
"""
Merge previous steps records
"""
if self.project_status_not_failed():
comp_logger.info('Initiating Image, Price, Text Confidence merger for project id: {}'.format(self.project_id))
self.output_dir_step6 = os.path.join(self.output_dir,settings.PROJECT_STEP_FOLDER_6)
helper.create_dir(self.output_dir_step6)
input_file_path = self.input_conf_file_path
output_file_path = os.path.join(self.output_dir_step6, settings.PROJECT_STEP_FILE_6)
try:
Step6_merger.main(project_id = self.project.id,
input_file_path = input_file_path,
output_file_path = output_file_path)
comp_logger.info('Image, Price, Text Confidence merger completed for project id: {}'.format(self.project_id))
except Exception as e:
comp_logger.info(str(e))
comp_logger.info('Image, Price, Text Confidence merger exception for project id: {}'.format(self.project_id))
self.update_project_status_failed('Image, Price, Text Confidence merger exception, '+ str(e))
traceback_error = traceback.format_exc()
comp_logger.info(traceback_error)
else:
comp_logger.info('Image, Price, Text Confidence merger skipped for project id: {} due to failed project status'.format(self.project_id))
def append_score_details(self):
if self.project_status_not_failed():
comp_logger.info('Initiating score details appender for project id: {}'.format(self.project_id))
output_dir_step7 = os.path.join(self.output_dir,settings.PROJECT_STEP_FOLDER_7)
helper.create_dir(output_dir_step7)
input_file_path = self.input_conf_file_path
step6_output_file_path = os.path.join(self.output_dir_step6, settings.PROJECT_STEP_FILE_6)
self.step7_output_file_path = os.path.join(output_dir_step7, settings.PROJECT_STEP_FILE_7.format(self.project_id))
try:
Step7_append_score_details.main(project_id=self.project_id,
client_input_file=input_file_path,
step6_output_file=step6_output_file_path,
output_file= self.step7_output_file_path)
comp_logger.info('Score Details appender completed for project id: {}'.format(self.project_id))
self.project.status = 'Completed'
self.project.save()
self.change_project_status('8')
except Exception as e:
comp_logger.info(str(e))
comp_logger.info('Score Details appender exception for project id: {}'.format(self.project_id))
self.update_project_status_failed('Score Details appender exception, '+ str(e))
traceback_error = traceback.format_exc()
comp_logger.info(traceback_error)
else:
comp_logger.info('Score Details appender skipped for project id: {} due to failed project status'.format(self.project_id))
def price_difference_calculation(self):
if self.project_status_not_failed():
comp_logger.info('Initiating Price Diff Calculation for project id: {}'.format(self.project_id))
output_dir_step4 = os.path.join(self.output_dir,settings.PROJECT_STEP_FOLDER_4)
helper.create_dir(output_dir_step4)
input_file_path = self.input_conf_file_path
output_file_path = os.path.join(output_dir_step4, settings.PROJECT_STEP_FILE_4)
if not os.path.exists(output_file_path):
try:
# Step4_PriceDiff_Calculation.main(input_file_path, output_file_path)
price_diff_calculator = PriceDiffereCalculator(input_file_path = input_file_path,
output_file_path = output_file_path)
price_diff_calculator.main()
comp_logger.info('Price Diff Calculation completed for project id: {}'.format(self.project_id))
self.change_project_status('6')
except Exception as e:
comp_logger.info(str(e))
comp_logger.info('Price Diff Calculation exception for project id: {}'.format(self.project_id))
self.update_project_status_failed('Price Diff Calculation exception, '+ str(e))
traceback_error = traceback.format_exc()
comp_logger.info(traceback_error)
else:
comp_logger.info('Price Differece calculation already completed for project id: {}. Restart process'.format(self.project_id))
else:
comp_logger.info('Price Diff Calculation skipped for project id: {} due to failed project status'.format(self.project_id))
def text_confidence_generation(self):
if self.project_status_not_failed():
comp_logger.info('Initiating Text Confidence computation for project id: {}'.format(self.project_id))
output_dir_step5 = os.path.join(self.output_dir,settings.PROJECT_STEP_FOLDER_5)
helper.create_dir(output_dir_step5)
input_file_path = self.input_conf_file_path
output_file_path = os.path.join(output_dir_step5, settings.PROJECT_STEP_FILE_5)
if not os.path.exists(output_file_path):
try:
# Step5_TextConfidence_Generation.main(input_file_path, output_file_path)
text_conf_generator = TextConfidenceGenerator(input_file_path = input_file_path,
output_file_path = output_file_path)
text_conf_generator.main()
comp_logger.info('Text confidence Computation completed for project id: {}'.format(self.project_id))
self.change_project_status('7')
except Exception as e:
comp_logger.info(str(e))
comp_logger.info('Text confidence Computation exception for project id: {}'.format(self.project_id))
self.update_project_status_failed('Text confidence Computation exception, '+ str(e))
traceback_error = traceback.format_exc()
comp_logger.info(traceback_error)
else:
comp_logger.info('Text confidence computation already completed for project id: {}. Restart process'.format(self.project_id))
else:
comp_logger.info('Text confidence Computation skipped for project id: {} due to failed project status'.format(self.project_id))
def get_base_file(self):
comp_logger.info('Initiating Step 0 for project id: {}'.format(self.project_id))
self.output_dir_step0 = os.path.join(self.output_dir,settings.PROJECT_STEP_FOLDER_0)
helper.create_dir(self.output_dir_step0)
try:
result_image_mapper_final_file_path = os.path.join(self.image_download_main_dir,
settings.PROJECT_RESULT_IMAGE_MAPPER_FINAL)
if os.path.exists(result_image_mapper_final_file_path):
self.result_image_file_path = result_image_mapper_final_file_path
Step0_get_required_fields_for_image_processing.main(input_file_path = self.input_conf_file_path,
output_dir = self.output_dir_step0,
output_file_name = settings.PROJECT_STEP_FILE_0,
result_image_file_path = self.result_image_file_path)
comp_logger.info('Step 0 completed for project id: {}'.format(self.project_id))
except Exception as e:
comp_logger.info(str(e))
comp_logger.info('Step 0 exception for project id: {}'.format(self.project_id))
self.update_project_status_failed('Step 0 exception, '+ str(e))
traceback_error = traceback.format_exc()
comp_logger.info(traceback_error)
def start_color_classification(self):
if self.project_status_not_failed():
comp_logger.info('Initiating Color Classification for project id: {}'.format(self.project_id))
output_dir_step2 = os.path.join(self.output_dir,settings.PROJECT_STEP_FOLDER_2)
helper.create_dir(output_dir_step2)
input_file_path = os.path.join(self.output_dir_step0, settings.PROJECT_STEP_FILE_0)
output_file_path = os.path.join(output_dir_step2, settings.PROJECT_STEP_FILE_2)
# print(self.image_download_main_dir)
if not os.path.exists(output_file_path):
try:
color_classifier = ImageColorClassifier(input_file_path = input_file_path,
output_file_path = output_file_path,
image_download_dir = self.image_download_main_dir)
color_classifier.start()
comp_logger.info('Color Classification completed for project id: {}'.format(self.project_id))
self.change_project_status('5')
except Exception as e:
comp_logger.info(str(e))
comp_logger.info('Color Classification exception for project id: {}'.format(self.project_id))
self.update_project_status_failed('Color Classification exception, '+ str(e))
traceback_error = traceback.format_exc()
comp_logger.info(traceback_error)
else:
comp_logger.info('Color classification already completed for project id: {}. Restart process'.format(self.project_id))
else:
comp_logger.info('Color Classification skipped for project id: {} due to failed project status'.format(self.project_id))
def download_images(self):
# Search Image Download Section
comp_logger.info('Search image downloader started for project id: {}'.format(self.project_id))
image_download_dir = os.path.join(self.image_download_main_dir,settings.PROJET_SEARCH_IMAGES_FOLDER)
helper.create_dir(image_download_dir)
try:
downloadSearchImageHelper.main(self.input_conf_file_path,image_download_dir)
comp_logger.info('Search image downloader completed for project id: {}'.format(self.project_id))
except Exception as e:
comp_logger.info(str(e))
comp_logger.info('Search image downloader exception for project id: {}'.format(self.project_id))
traceback_error = traceback.format_exc()
comp_logger.info(traceback_error)
# Result Image Download Section
comp_logger.info('Result image downloader started for project id: {}'.format(self.project_id))
image_download_dir = os.path.join(self.image_download_main_dir,settings.PROJET_RESULT_IMAGES_FOLDER)
helper.create_dir(image_download_dir)
try:
self.result_image_file_path = os.path.join(self.image_download_main_dir, settings.PROJECT_RESULT_IMAGE_MAPPER)
downloadResultImageHelper.main(self.input_conf_file_path,image_download_dir,self.result_image_file_path)
comp_logger.info('Result image downloader completed for project id: {}'.format(self.project_id))
self.change_project_status('2')
except Exception as e:
comp_logger.info(str(e))
comp_logger.info('Result image downloader exception for project id: {}'.format(self.project_id))
traceback_error = traceback.format_exc()
comp_logger.info(traceback_error)
def __init__(self, project, project_status):
# Thread.__init__(self)
self.project = project
self.project_status = project_status
self.project_id = self.project.id
# creating directory for project
project_dir = os.path.join(settings.PROJECT_DIR,str(self.project_id))
helper.create_dir(project_dir)
project_input_dir = os.path.join(project_dir,settings.PROJECT_PROCESS_INPUT_FOLDER)
helper.create_dir(project_input_dir)
# creating image directories
self.image_download_main_dir = os.path.join(settings.PROJECT_DIR, str(self.project_id),settings.PROJECT_IMAGES_FOLDER)
helper.create_dir(self.image_download_main_dir)
# self.input_conf_file_path = os.path.join(settings.MEDIA_ROOT,settings.INPUT_DIR,self.project.file)
# file_format = project.file_name.split('.')[-1]
# global_project_input_file_path = os.path.join(settings.FTP_ROOT_FOLDER_LOCATION,settings.PROJECT_INPUT_FILE_PATTERN.format(self.project_id, file_format))
global_project_input_file_path = os.path.join(settings.FTP_ROOT_FOLDER_LOCATION,self.project.file_name)
self.input_conf_file_path = os.path.join(project_input_dir,project.file_name)
if not os.path.exists(self.input_conf_file_path):
# copy remote project input file to local
self.copy_project_input_file(global_project_input_file_path, self.input_conf_file_path)
self.output_dir = os.path.join(settings.PROJECT_DIR, str(self.project_id),settings.PROJECT_STEP_FOLDER)
helper.create_dir(self.output_dir)
def start_imagenet_classification(self):
if self.project_status_not_failed():
comp_logger.info('Initiating Imagenet Classification for project id: {}'.format(self.project_id))
output_dir_step1 = os.path.join(self.output_dir,settings.PROJECT_STEP_FOLDER_1)
helper.create_dir(output_dir_step1)
input_file_path = os.path.join(self.output_dir_step0, settings.PROJECT_STEP_FILE_0)
try:
Step1_imagenet_classifier.main(input_file_path = input_file_path,
output_dir = output_dir_step1,
image_download_main_dir = self.image_download_main_dir,
project_id = self.project_id)
comp_logger.info('Imagenet Classification completed for project id: {}'.format(self.project_id))
self.change_project_status('4')
except Exception as e:
comp_logger.info(str(e))
comp_logger.info('Imagenet Classification exception for project id: {}'.format(self.project_id))
self.update_project_status_failed('Imagenet Classification exception, '+ str(e))
traceback_error = traceback.format_exc()
comp_logger.info(traceback_error)
else:
comp_logger.info('Imagenet Classification skipped for project id: {} due to failed project status'.format(self.project_id))
def __init__(self, project) -> None:
comp_logger.info("Initializing Process")
# initialize firefox driver
options = Options()
options.headless = True
self.driver = webdriver.Firefox(
options=options, executable_path=settings.GECKO_DRIVER_PATH)
# Process instance settings
self.project = project
self.project_id = self.project.id
self.query = self.project.search_query
self.user = self.project.user
self.search_image = os.path.join(settings.BASE_DIR,
self.project.search_image)
self.project_dir = os.path.join(settings.PROCESS_FOLDER,
str(self.project_id))
self.recommendation_html_file_path = os.path.join(
self.project_dir, settings.RECOMMEDATION_HTML_FILE)
self.recommendation_html_screenshot = os.path.join(
self.project_dir, settings.RECOMMENDATION_SCREENSHOT)
self.image_dir = os.path.join(self.project_dir, settings.IMAGES_FOLDER)
self.page_source_dir = os.path.join(self.project_dir,
settings.PAGE_SOURCE_FOLDER)
# Process image file path to be used
self.search_image_path = os.path.join(self.project_dir,
settings.SOURCE_IMAGE_PATH)
helper.create_dir(self.project_dir)
helper.create_dir(self.image_dir)
helper.create_dir(self.page_source_dir)
helper.copy_file(self.search_image, self.search_image_path)
self.top_n_picks_recommendation = ""
self.catalog_df = pd.DataFrame(
columns=settings.SERP_PRODUCT_CATALOG_HEADERS)
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'prospect_pla_showcase.urls'
LOGGER_LIST = ['prospect']
LOGGER_CONTENT= {
'handlers': ['file_handler','console'],
'level': 'DEBUG',
'propagate': True,
}
# Logging details
helper.create_dir(os.path.join(BASE_DIR,'logs'))
LOGGING = {
'version': 1,
'disable_existing_loggers': True,
'filters': {
'require_debug_false': {
'()': 'django.utils.log.RequireDebugFalse',
},
'require_debug_true': {
'()': 'django.utils.log.RequireDebugTrue',
},
},
'formatters': {
'server_format': {
# 'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s',
