def queryWebsite(url,cookie_header,raw_data,method_type):
#print(url)
#print(raw_data)
config_dict=ConfigReader.get_confic_dict()
return_list=[]
price_vendor_list=[]
read_response = Utils.makeRequestUrl(url,cookie_header,raw_data,method_type)
#print(read_response.text)
if read_response and read_response != Constants.EXCEPTION_QUERY:
#print(read_response)
soup = BeautifulSoup(read_response.text,'html.parser')
extracted_data=soup.find(id=config_dict[Constants.Healthy_Buddha]['id_value']).find_all('option')
#print(extracted_data)
if extracted_data :
#print(extracted_data)
data = extracted_data[Constants.NUM_0]
#print(data)
sku_unit=str(data).split(Constants.DASH+Constants.SPACE+Constants.LESSER_OP)[Constants.NUM_0].split(Constants.GREATER_OP)[Constants.NUM_1]
sku_value=str(data).split(Constants.DASH+Constants.SPACE+Constants.LESSER_OP)[Constants.NUM_1].split(Constants.GREATER_OP)[Constants.NUM_2].split(Constants.LESSER_OP)[Constants.NUM_0]
price_vendor_list.append(Constants.Healthy_Buddha.replace(Constants.UNDERSCORE,Constants.SPACE))
price_vendor_list.append(sku_value.strip())
#print(price_vendor_list)
return_list.append(price_vendor_list)
price_vendor_list=[]
price_vendor_list.append(Constants.Healthy_Buddha.replace(Constants.UNDERSCORE,Constants.SPACE)+Constants.SPACE+Constants.STR_SKU)
price_vendor_list.append(sku_unit)
#print(price_vendor_list)
return_list.append(price_vendor_list)
if config_dict[Constants.Healthy_Buddha]['discounted_is_required'] == str(True):
print("if True then will add the discounted price also not required in this case, you can add later")
#print(return_list)
return return_list
def getUrlWithHeaderRawData(vendor):
#print(vendor)
config_dict = ConfigReader.get_confic_dict()
url = config_dict[Constants.Namdhari_s][
'http_scheme'] + Constants.COLON + Constants.DOUBLEFORWARDSLASH + config_dict[
Constants.Namdhari_s]['base_url'] + vendor[
'product_query'] + Constants.FORWARDSLASH + vendor[
'product_id'] + Constants.DOT + config_dict[
Constants.Namdhari_s]['url_extension']
#print(vendor['product_vpid'])
#print(vendor)
try:
#print(vendor['product_vpid'])
if vendor['product_vpid'] != "NA":
url += Constants.QUESTION_MARK + "vpid" + Constants.EQUALS + vendor[
'product_vpid']
except KeyError:
print(
"product_vpid is not found, normal url will be used for quering "
+ vendor['product_name'] + " || vendor is " +
vendor['vendor_name'])
#print(url)
cookie_header = requests.cookies.RequestsCookieJar()
cookie_header.set('Site_Config', vendor['location_query'])
raw_data = Constants.NONE
return url, cookie_header, raw_data
def getUrlWithHeaderRawData(vendor):
#print(vendor)
config_dict=ConfigReader.get_confic_dict()
url=config_dict[Constants.Healthy_Buddha]['http_scheme']+Constants.COLON+Constants.DOUBLEFORWARDSLASH+config_dict[Constants.Healthy_Buddha]['base_url']+vendor['product_query']
cookie_header=Constants.NONE
raw_data=Constants.NONE
return url,cookie_header,raw_data
def queryWebsite(url, cookie_header, raw_data, method_type):
#print(raw_data)
config_dict = ConfigReader.get_confic_dict()
return_list = []
price_vendor_list = []
data = Utils.makeRequestUrl(url, cookie_header, raw_data,
method_type).json()
#print(data)
if data:
price_vendor_list.append(
Constants.Townness.replace(Constants.UNDERSCORE,
Constants.SPACE))
price_vendor_list.append(data[Constants.NUM_0][config_dict[
Constants.Townness]['mrp_syntax']])
return_list.append(price_vendor_list)
if config_dict[
Constants.Townness]['discounted_is_required'] == str(True):
#print("if True then will add the discounted price also not required in this case, you can add later")
price_vendor_list = []
price_vendor_list.append(
Constants.Townness_Discounted.replace(
Constants.UNDERSCORE, Constants.SPACE))
price_vendor_list.append(data[Constants.NUM_0][config_dict[
Constants.Townness]['sp_syntax']])
return_list.append(price_vendor_list)
return return_list
def getUrlWithHeaderRawData(vendor):
#print(vendor)
config_dict=ConfigReader.get_confic_dict()
url=config_dict[Constants.Gournet_Garden]['http_scheme']+Constants.COLON+Constants.DOUBLEFORWARDSLASH+config_dict[Constants.Big_basket]['base_url']+vendor['product_id']+Constants.FORWARDSLASH+vendor['product_query']+Constants.FORWARDSLASH
cookie_header=requests.cookies.RequestsCookieJar()
cookie_header.set('_bb_cid',vendor['location_query'])
raw_data=Constants.NONE
return url,cookie_header,raw_data
def recDataPersist(data_dict):
#print(data_dict)
config_dict=ConfigReader.get_confic_dict()
status,fileName=ModelMonthlyBucketDataPersister.createExcelFile()
if status == Constants.STR_SUCCESS:
ModelMonthlyBucketDataPersister.loadExcelAddSheet(fileName,config_dict[Constants.STRING_APP]['daily_excel_sheet_one_name'].replace(Constants.UNDERSCORE,Constants.SPACE))
ModelMonthlyBucketDataPersister.firt2RowPersistInSheet(fileName,config_dict[Constants.STRING_APP]['daily_excel_sheet_one_name'].replace(Constants.UNDERSCORE,Constants.SPACE))
ModelMonthlyBucketDataPersister.writeDictDataInExcelModelMontlyBasket(data_dict,fileName,config_dict[Constants.STRING_APP]['daily_excel_sheet_one_name'].replace(Constants.UNDERSCORE,Constants.SPACE))
def getUrlWithHeaderRawData(vendor):
#print(vendor)
config_dict = ConfigReader.get_confic_dict()
url = config_dict[Constants.Gournet_Garden][
'http_scheme'] + Constants.COLON + Constants.DOUBLEFORWARDSLASH + config_dict[
Constants.Townness]['base_url']
cookie_header = Constants.NONE
raw_data = "categoryid=" + vendor[
'category_id'] + "&productid=" + vendor[
'product_id'] + "&type=getproduct"
return url, cookie_header, raw_data
def AppendFileName():
config_dict=ConfigReader.get_confic_dict()
excelNameAppender=config_dict[Constants.STRING_APP]['daily_excel_base_path']+config_dict[Constants.STRING_APP]['daily_excel_file_name_prefix'].replace(Constants.UNDERSCORE,Constants.SPACE)+Constants.UNDERSCORE
if config_dict[Constants.STRING_APP]['daily_excel_appender_type'] == Constants.STRING_DATE:
excelNameAppender=excelNameAppender+datetime.today().strftime(config_dict[Constants.STRING_APP]['daily_excel_type_format'])
else:
print("If appender for the file name changed will add implemetation here in future")
excelNameAppender=excelNameAppender+Constants.DOT+config_dict[Constants.STRING_APP]['daily_excel_extension']
return excelNameAppender
def queryWebsite(url, cookie_header, raw_data, method_type):
#print(url)
#print(raw_data)
config_dict = ConfigReader.get_confic_dict()
return_list = []
price_vendor_list = []
read_response = Utils.makeRequestUrl(url, cookie_header, raw_data,
method_type)
#print(r.text)
if read_response and read_response != Constants.EXCEPTION_QUERY:
#print(read_response)
soup = BeautifulSoup(read_response.text, features="html.parser")
parsed_txt = soup.find_all(
config_dict[Constants.Organic_World]['tag_value'])
#print(parsed_txt)
extracted_data = Constants.NONE
for txt in parsed_txt:
if re.search('var meta', txt.text):
#print(txt.text)
extracted_data = txt.text
break
if extracted_data != Constants.NONE:
data_str = extracted_data.split("var meta = ")[
Constants.NUM_1].split(';\nfor')[Constants.NUM_0]
data = json.loads(data_str)
any_one_variant = data['product']['variants'][Constants.NUM_0]
sku_value = any_one_variant['price'] / 100
sku_unit = any_one_variant['public_title']
if sku_unit is None:
sku_unit = any_one_variant['name'].split(
Constants.COMMA)[Constants.NUM_1]
price_vendor_list.append(
Constants.Organic_World.replace(Constants.UNDERSCORE,
Constants.SPACE))
price_vendor_list.append(sku_value)
#print(price_vendor_list)
return_list.append(price_vendor_list)
#print(sku_value+Constants.SPACE+sku_unit)
price_vendor_list = []
price_vendor_list.append(
Constants.Organic_World.replace(Constants.UNDERSCORE,
Constants.SPACE) +
Constants.SPACE + Constants.STR_SKU)
price_vendor_list.append(sku_unit)
#print(price_vendor_list)
return_list.append(price_vendor_list)
if config_dict[Constants.Organic_World][
'discounted_is_required'] == str(True):
print(
"if True then will add the discounted price also not required in this case, you can add later"
)
print(return_list)
return return_list
def __init__(self):
config_dict = ConfigReader.get_confic_dict()
self.HOST_NAME = config_dict[Constants.SMTP_DICT][Constants.HOST_NAME]
self.HOST_PORT = config_dict[Constants.SMTP_DICT][Constants.HOST_PORT]
self.ALLOWSSL = config_dict[Constants.SMTP_DICT][
Constants.SMTP_ALLOWSSL]
self.ALLOWSTARTTLS = config_dict[Constants.SMTP_DICT][
Constants.SMTP_ALLOWSTARTTLS]
self.SENDER = config_dict[Constants.SMTP_DICT][Constants.SMTP_SENDER]
self.PASSWORD = str(
base64.b64decode(
config_dict[Constants.SMTP_DICT][Constants.SMTP_PASSWORD]),
"utf-8")
def queryWebsite(url, cookie_header, raw_data, method_type):
#print(raw_data)
config_dict = ConfigReader.get_confic_dict()
return_list = []
price_vendor_list = []
read_response = Utils.makeRequestUrl(url, cookie_header, raw_data,
method_type)
#print(r.text)
if read_response and read_response != Constants.EXCEPTION_QUERY:
soup = BeautifulSoup(read_response.text, 'html.parser')
parsed_txt = soup.find(
id=config_dict[Constants.Gournet_Garden]['script_parser_id'])
#print(parsed_json)
if parsed_txt is not None:
data = json.loads(parsed_txt.text.strip())
#print(data)
price_vendor_list.append(
Constants.Gournet_Garden.replace(Constants.UNDERSCORE,
Constants.SPACE))
price_vendor_list.append(data[config_dict[
Constants.Gournet_Garden]['price_syntax_in_json']] /
Constants.NUM_100)
return_list.append(price_vendor_list)
parsed_txt_name_with_sku = soup.find(
config_dict[Constants.Gournet_Garden]['meta_tag_value'],
property=config_dict[
Constants.Gournet_Garden]['match_name_content'])
sku_parse = str(parsed_txt_name_with_sku[config_dict[
Constants.Gournet_Garden]['sku_syntax_in_json']]).split(
Constants.DASH)
length_sku_list = len(sku_parse)
sku_list_value = sku_parse[Constants.NUM_1:length_sku_list]
#print(sku_parse[Constants.NUM_1:length_sku_list])
sku_value = Constants.SPACE.join(map(str, sku_list_value))
#print(sku_value)
price_vendor_list = []
price_vendor_list.append(
Constants.Gournet_Garden.replace(Constants.UNDERSCORE,
Constants.SPACE) +
Constants.SPACE + Constants.STR_SKU)
price_vendor_list.append(sku_value)
return_list.append(price_vendor_list)
if config_dict[Constants.Gournet_Garden][
'discounted_is_required'] == str(True):
print(
"if True then will add the discounted price also not required in this case, you can add later"
)
return return_list
def queryWebsite(url, cookie_header, raw_data, method_type):
#print(raw_data)
config_dict = ConfigReader.get_confic_dict()
return_list = []
price_vendor_list = []
read_response = Utils.makeRequestUrl(url, cookie_header, raw_data,
method_type)
#print(read_response)
if read_response and read_response != Constants.EXCEPTION_QUERY:
soup = BeautifulSoup(read_response.text, 'html.parser')
data = soup.find(
config_dict[Constants.Nature_s_Basket]['element_type'], {
Constants.STR_ID:
config_dict[Constants.Nature_s_Basket]['element_id']
})[Constants.STR_VALUE]
#print(data)
price_vendor_list.append(
Constants.Nature_s_Basket.replace(Constants.UNDERSCORE,
Constants.SPACE))
price_vendor_list.append(data)
return_list.append(price_vendor_list)
sku_data = soup.find(
config_dict[Constants.Nature_s_Basket]['sku_element_type'], {
Constants.STR_CLASS:
config_dict[Constants.Nature_s_Basket]['sku_element_id']
})
sku_list = sku_data.get_text().split(Constants.DASH)
sku_list_len = len(sku_list)
sku_value = sku_list[sku_list_len - Constants.NUM_1]
#print(sku_value)
price_vendor_list = []
price_vendor_list.append(
Constants.Nature_s_Basket.replace(
Constants.UNDERSCORE, Constants.SPACE) + Constants.SPACE +
Constants.STR_SKU)
price_vendor_list.append(sku_value)
return_list.append(price_vendor_list)
if config_dict[Constants.Nature_s_Basket][
'discounted_is_required'] == str(True):
print(
"if True then will add the discounted price also not required in this case, you can add later"
)
return return_list
def start(self):
config_dict = ConfigReader.get_confic_dict()
raw_file_path = config_dict[
Constants.STRING_APP]['info_file_basepath_app'] + config_dict[
Constants.STRING_APP]['info_file_path_app']
#print(raw_file_path)
dict_data = defaultdict(list)
data = Utils.loadDataFromFile(raw_file_path)
if data != Constants.EXCEPTION_LOAD:
for info in data[Constants.INFO_DETAIL]:
#print(info)
data_dict_info = {}
product_name = info[Constants.STR_PRODUCT_NAME]
data_dict_info[Constants.STR_PRODUCT_NAME] = product_name
product_type = info[Constants.STR_PRODUCT_TYPE]
#data_dict_info[Constants.STR_PRODUCT_TYPE]=product_type
sku_g = info[Constants.STR_SKU_G]
data_dict_info[Constants.STR_SKU_G] = sku_g
sku_u = info[Constants.STR_SKU_U]
data_dict_info[Constants.STR_SKU_U] = sku_u
vendor_list_in_dict = []
for vendor in info[Constants.STR_VENDOR_LIST]:
vendor_name = vendor[Constants.STR_V_NAME]
vendor_price_list = self.vendor_url_defination_caller(
vendor)
#print(url+""+cookie_header+""+raw_data+""+str(price))
#print(vendor_name)
if vendor_price_list:
for v_p in vendor_price_list:
vendor_list_in_dict.append(
(v_p[Constants.NUM_0] + Constants.DOLLAR +
str(v_p[Constants.NUM_1])).split(
Constants.DOLLAR))
print(str(vendor_list_in_dict) + "Data added")
data_dict_info[Constants.STR_VENDOR_LIST] = vendor_list_in_dict
vendor_list_in_dict = []
dict_data[product_type].append(data_dict_info)
data_dict_info = {}
else:
print(data)
ModelMonthlyBucketDataPersister.recDataPersist(dict_data)
def getUrlWithHeaderRawData(vendor):
#print(vendor)
config_dict = ConfigReader.get_confic_dict()
url = config_dict[Constants.Gournet_Garden][
'http_scheme'] + Constants.COLON + Constants.DOUBLEFORWARDSLASH + config_dict[
Constants.Gournet_Garden]['base_url'] + vendor[
'product_query'] + Constants.QUESTION_MARK + Constants.WRD_CITY + Constants.EQUALS + vendor[
'location_query']
try:
#print(vendor['product_vpid'])
if vendor['v_id'] != "NA":
url += Constants.AMPERCENT + "variant" + Constants.EQUALS + vendor[
'v_id']
except KeyError:
print(
"product_vpid is not found, normal url will be used for quering "
+ vendor['product_name'] + " || vendor is " +
vendor['vendor_name'])
cookie_header = Constants.NONE
raw_data = Constants.NONE
return url, cookie_header, raw_data
def queryWebsite(url,cookie_header,raw_data,method_type):
#print(raw_data)
config_dict=ConfigReader.get_confic_dict()
return_list=[]
price_vendor_list=[]
read_response = Utils.makeRequestUrl(url,cookie_header,raw_data,method_type)
#print(r.text)
if read_response and read_response != Constants.EXCEPTION_QUERY:
soup = BeautifulSoup(read_response.text,'html.parser')
parsed_txt=soup.findAll(config_dict[Constants.Big_basket]['script_tag_value'])
#print(parsed_txt)
for txt in parsed_txt:
if re.search(config_dict[Constants.Big_basket]['match_price_content'],txt.text):
print("1"+str(url))
#print(txt.text)
data_txt_split=txt.text.split(config_dict[Constants.Big_basket]['split_matched_content'].replace(Constants.DOLLAR,Constants.SPACE))
extracted_data=data_txt_split[Constants.NUM_1].strip()
#print(extracted_data)
data = json.loads(json.loads(extracted_data))
#print(data)
price_vendor_list.append(Constants.Big_basket.replace(Constants.UNDERSCORE,Constants.SPACE))
price_vendor_list.append(data[config_dict[Constants.Big_basket]['json_product_index_name']][config_dict[Constants.Big_basket]['json_variants_index_name']][Constants.NUM_0][config_dict[Constants.Big_basket]['mrpprice_syntax_in_json']])
return_list.append(price_vendor_list)
price_vendor_list=[]
price_vendor_list.append(Constants.Big_basket.replace(Constants.UNDERSCORE,Constants.SPACE)+Constants.SPACE+Constants.STR_SKU)
price_vendor_list.append(data[config_dict[Constants.Big_basket]['json_product_index_name']][config_dict[Constants.Big_basket]['json_variants_index_name']][Constants.NUM_0][config_dict[Constants.Big_basket]['json_sku_name']])
return_list.append(price_vendor_list)
if config_dict[Constants.Big_basket]['discounted_is_required'] == str(True):
#print("if True then will add the discounted price also not required in this case, you can add later")
price_vendor_list=[]
price_vendor_list.append(Constants.Big_basket_discounted.replace(Constants.UNDERSCORE,Constants.SPACE))
price_vendor_list.append(data[config_dict[Constants.Big_basket]['json_product_index_name']][config_dict[Constants.Big_basket]['json_variants_index_name']][Constants.NUM_0][config_dict[Constants.Big_basket]['spprice_syntax_in_json']])
return_list.append(price_vendor_list)
break
return return_list
def queryWebsite(url, cookie_header, raw_data, method_type):
#print(url)
#print(raw_data)
config_dict = ConfigReader.get_confic_dict()
v_product_id = Simpli_Namdharis_Defination.find_V_ProductID(
url, config_dict)
#print(v_product_id)
return_list = []
price_vendor_list = []
read_response = Utils.makeRequestUrl(url, cookie_header, raw_data,
method_type)
#print(r.text)
if read_response and read_response != Constants.EXCEPTION_QUERY:
soup = BeautifulSoup(read_response.text, 'html.parser')
parsed_txt = soup.findAll(
config_dict[Constants.Namdhari_s]['script_tag_value'])
#print(parsed_txt)
data_txt_list = []
for txt in parsed_txt:
if re.search(
config_dict[Constants.Namdhari_s]
['match_price_content'], txt.text):
#print(txt)
data_txt_split = txt.text.split(config_dict[
Constants.Namdhari_s]['split_matched_content'].replace(
Constants.DOLLAR, Constants.SPACE))
#extracted_data=data_txt_split[Constants.NUM_1].strip()\
data_txt_list.append(data_txt_split)
#print(extracted_data)
#print(data_txt_list[Constants.NUM_0][Constants.NUM_1])
#print(data_txt_list)
extracted_data = data_txt_list[Constants.NUM_0][Constants.NUM_1]
#print(extracted_data)
data = json.loads(str(extracted_data))
variant_data = data[config_dict[
Constants.Namdhari_s]['json_product_index_name']][config_dict[
Constants.Namdhari_s]['json_variants_index_name']]
if variant_data is not None:
#print(variant_data)
data_variants = json.loads(variant_data)
#print(data_variants)
append_price_data = None
sku_value = None
if v_product_id == Constants.NONE:
append_price_data = data_variants[
Constants.NUM_0][config_dict[Constants.Namdhari_s]
['mrpprice_syntax_in_json']]
sku_value = data_variants[Constants.NUM_0][config_dict[
Constants.Namdhari_s]['sku_variant_index_name']][
config_dict[Constants.Namdhari_s]
['sku_variant_weight_name']]
#print(sku_value)
else:
for data_v in data_variants:
#print("variants"+str(data_v[config_dict[Constants.Namdhari_s]['variants_match_content']]))
if str(data_v[config_dict[Constants.Namdhari_s]
['variants_match_content']]) == str(
v_product_id):
#print("matched variant_product_id")
append_price_data = data_v[
config_dict[Constants.Namdhari_s]
['mrpprice_syntax_in_json']]
sku_value = data_v[
config_dict[Constants.Namdhari_s]
['sku_variant_index_name']][
config_dict[Constants.Namdhari_s]
['sku_variant_weight_name']]
#print(sku_value)
break
#print("append data"+str(append_price_data))
if append_price_data is not None:
price_vendor_list.append(
Constants.Namdhari_s.replace(Constants.UNDERSCORE,
Constants.SPACE))
price_vendor_list.append(append_price_data)
#print(price_vendor_list)
return_list.append(price_vendor_list)
if sku_value is not None:
price_vendor_list = []
price_vendor_list.append(
Constants.Namdhari_s.replace(Constants.UNDERSCORE,
Constants.SPACE) +
Constants.SPACE + Constants.STR_SKU)
price_vendor_list.append(sku_value)
#print(price_vendor_list)
return_list.append(price_vendor_list)
if config_dict[Constants.
Namdhari_s]['discounted_is_required'] == str(True):
print(
"if True then will add the discounted price also not required in this case, you can add later"
)
return return_list
