def __AggregateFileS(self):
self.progress_bar_count = 0
print("")
CommonUtilities.progressBar(self.progress_bar_count,
len(self.lk_path_dict.keys()),
status="Loading ..")
new_merged_file = self.__AggregateFiles(self.lk_path_dict.values())
CommonUtilities.writeDictToJson(
new_merged_file, f"{self.dst_dir_name}/big_cluster.json")
def RunInteration(self):
print(f"[{type(self).__name__}]Running ---> RunInteration")
self.exc_start_time = datetime.datetime.now()
self.__findOrCreateDir("")
self.__AggregateFileSameSourceAndSpect()
CommonUtilities.writeDictToJson(self.lk_1_path_dict, self.dst_lk_path_dict)
self.exc_end_time = datetime.datetime.now()
self.__printStats()
def __AggregateAttributesFile(self):
for objectSpect, objectSources in self.lk_path_dict.items():
for sources, filepath in objectSources.items():
self.__findOrCreateDir(sources)
self.__current_json_data = CommonUtilities.loadJsonFile(
f"{self.src_dir_name}/{filepath}")
self.__AggregateAttributes()
CommonUtilities.writeDictToJson(
self.__current_json_data,
f"{self.dst_dir_name}/{filepath}.json")
def __makeCollisionDictionary(self):
print(f"[{type(self).__name__}]Running ---> makeCollisionDictionary")
self.exc_start_time = datetime.datetime.now()
for objectSpect, objectValues in self.filePathData.items():
for fileSource, filesList in objectValues.items():
for filepath in filesList:
curr_jsonData = CommonUtilities.loadJsonFile(
f"{self.srcDir}/{filepath}")
self.__mergeFileWithCollisionDict(curr_jsonData)
CommonUtilities.writeDictToJson(self.collision_dict, self.outFile)
self.exc_end_time = datetime.datetime.now()
def makeJsonPathFile_EXT(self):
print(f"[{type(self).__name__}]Running ---> makeJsonPathFile_EXT")
for dir in os.listdir(BASE_SOURCE_DIR):
for file in os.listdir(f"{BASE_SOURCE_DIR}/{dir}/"):
filename = file.split(".")[0]
rs = f"{dir}/{filename}"
if not rs in self.resources_founded:
self.dictionary_ext_path[rs] = {
CommonUtilities.getSourceFromPath(rs): [rs]
}
self.total_file_external += 1
CommonUtilities.writeDictToJson(self.dictionary_ext_path,
SOURCES_BASE_EXT_DICT)
def get_symbols_from_library(libraryPath):
symbolNumber = 0
all_object_files_symbols = CommonUtilities.execute_command(
"nm -extern-only " + libraryPath).split('\n\n')
for object_file_symbols in all_object_files_symbols:
symbolNumber += handle_object_files_symbols(object_file_symbols)
return symbolNumber
def __AggregateFiles(self, filesList):
mergedData = {}
for filepath in filesList:
jsdata = CommonUtilities.loadJsonFile(
f"{self.src_dir_name}/{filepath}")
for attrName, attrValue in jsdata.items():
if not attrName in mergedData.keys():
mergedData[attrName] = []
mergedData[attrName] += attrValue
self.progress_bar_count += 1
CommonUtilities.progressBar(self.progress_bar_count,
len(self.lk_path_dict.keys()),
status=f"Agg: {filepath}")
return mergedData
def __AggregateAttributesFile(self):
progressCount = 0
print("")
CommonUtilities.progressBar(progressCount,
len(self.lk_path_dict.keys()),
status="Loading ..")
for objectSpect, filepath in self.lk_path_dict.items():
progressCount += 1
CommonUtilities.progressBar(progressCount,
len(self.lk_path_dict.keys()),
status=f"Agg: {objectSpect}")
self.__current_json_data = CommonUtilities.loadJsonFile(
f"{self.src_dir_name}/{filepath}")
#print(f"{self.src_dir_name}/{filepath}")
###Appena viene caricato il file per il l' aggregazione creo i dizionari dinamici
self.dym_dict_local = SecondIterationDictionary(
self.__current_json_data)
self.dym_dict_local.Load()
##Istanzio nella class i dinionari
self.coll_sim_din = self.dym_dict_local.dyn_col_sim
self.coll_inv_din = self.dym_dict_local.col_inv
self.__AggregateAttributes()
CommonUtilities.writeDictToJson(
self.__current_json_data,
f"{self.dst_dir_name}/{filepath}.json")
self.__dynDict.save()
def makeJsonPathFile_LK(self):
print(f"[{type(self).__name__}]Running ---> makeJsonPathFile_LK")
wb = xlrd.open_workbook(self.lkFileName)
sheet = wb.sheet_by_index(0)
for x in range(1, sheet.nrows):
CommonUtilities.progressBar(x, sheet.nrows)
row = CommonUtilities.splitOnComma(sheet.cell_value(x, 0))
left_r, righ_r = self.__getLeftRightElment(row)
source_lr = CommonUtilities.getSourceFromPath(left_r)
source_rr = CommonUtilities.getSourceFromPath(righ_r)
if not left_r in self.dictionary_lk_path.keys(
) and not left_r in self.resources_founded:
self.dictionary_lk_path[left_r] = {source_lr: [left_r]}
self.resources_founded.append(left_r)
self.total_object_linked += 1
self.total_file_linkage += 1
if not righ_r in self.resources_founded:
if not source_rr in self.dictionary_lk_path[left_r].keys():
self.dictionary_lk_path[left_r][source_rr] = []
self.dictionary_lk_path[left_r][source_rr].append(righ_r)
self.resources_founded.append(righ_r)
self.total_file_linkage += 1
CommonUtilities.writeDictToJson(self.dictionary_lk_path,
SOURCES_BASE_LK_DICT)
def getFlowList():
"""
从数据库获取所有的工作流列表
:return:返回{"name":FlowName,"value":FlowID}
"""
raw = rawSql.Raw_sql()
raw.sql = "SELECT FlowID AS value, FlowName AS name FROM RMI_WORK_FLOW WITH(NOLOCK)"
res, columns = raw.query_all(needColumnName=True)
return CommonUtilities.translateQueryResIntoDict(columns, res)
def __AggregateFileSameSourceAndSpect(self):
for objectSpect, objectSources in self.lk_path_dict.items():
self.lk_1_path_dict[objectSpect] = {}
for sources, filespath in objectSources.items():
self.__findOrCreateDir(sources)
self.lk_1_path_dict[objectSpect][
sources] = f"{sources}/{self.newFileNameID}"
new_merged_file = self.__AggregateFiles(filespath)
dym_dict = FirstIterationDictionary(new_merged_file,
self.newFileNameID)
dym_dict.Load()
CommonUtilities.writeDictToJson(
new_merged_file,
f"{self.dst_dir_name}/{sources}/{self.newFileNameID}.json")
self.newFileNameID += 1
def cleanDataSet(self):
print(f"[{type(self).__name__}]Running ---> cleanDataSet")
self.exc_start_time = datetime.datetime.now()
for object_spect, object_val in self.cm_path_dict.items():
for source_name, source_files in object_val.items():
self.__findOrCreateDir(source_name)
for file_path in source_files:
self.__cleanSingleFile(source_name, file_path)
CommonUtilities.writeDictToJson(self.discarded_info_pool,
f"{DROPPED_ATTRIBUTES_FILES}")
CommonUtilities.writeDictToJson(self.composite_value_pool,
f"{COMPOSITE_ATTRIBUTES_FILES}")
self.exc_end_time = datetime.datetime.now()
self.__printStats()
def __aggregateSim(self, arrSim):
##Per ogni coppia chiave --> chiavi_simili
for key, keySim in arrSim:
for k2 in keySim:
###Se la chiave e' sinonimo e non la stessa
if not key == k2:
###Se k2 esiste potrebbe essere stata accorpata
if k2 in self.__current_json_data.keys(
) and key in self.__current_json_data.keys():
#Costruisco due array contenenti gli indici numerici dei valori comuni delle due chiavi
currmtc1, currmtc2 = CommonUtilities.matchValues(
self.__current_json_data[key],
self.__current_json_data[k2])
if len(currmtc2) > 0:
for idx in range(0, len(currmtc2)):
##Calcolo le occorrenze delle chiavi rispetto al valore X ogni valore in comune
s1, s2 = CommonUtilities.scoreBinaryValueMatch(
key, k2, self.__collisionInvDict[
self.__current_json_data[k2][
currmtc2[idx]][1]])
if s1 >= s2:
item = self.__current_json_data[k2].pop(
currmtc2[idx])
item.append(k2)
self.__current_json_data[key].append(item)
##Dopo aver fatto il pop se la chiave non contiene valori la elimino
if len(self.__current_json_data[k2]) < 1:
del self.__current_json_data[k2]
for idx in range(0, len(currmtc1)):
s1, s2 = CommonUtilities.scoreBinaryValueMatch(
key, k2, self.__collisionInvDict[
self.__current_json_data[key][
currmtc1[idx]][1]])
if s1 < s2:
item = self.__current_json_data[key].pop(
currmtc1[idx])
item.append(key)
self.__current_json_data[k2].append(item)
##Dopo aver fatto il pop se la chiave non contiene valori la elimino
if len(self.__current_json_data[key]) < 1:
del self.__current_json_data[key]
def __AggregateFiles(self, filesList):
mergedData = {}
for filepath in filesList:
jsdata = CommonUtilities.loadJsonFile(f"{self.src_dir_name}/{filepath}")
for attrName, attrValue in jsdata.items():
if not attrName in mergedData.keys():
mergedData[attrName] = []
mergedData[attrName].append((filepath, attrValue))
return mergedData
def __makeCollisionInvDictionary(self):
print(
f"[{type(self).__name__}]Running ---> __makeCollisionInvDictionary"
)
self.exc_start_time = datetime.datetime.now()
for keyAttribute, valueAttributeList in self.__collision_dict.items():
for valueAttributeCount, valueAttribute in valueAttributeList[
"value_list"]:
if not valueAttribute in self.collision_inv_dict.keys():
self.collision_inv_dict[valueAttribute] = {
'attribute_list': []
}
self.collision_inv_dict[valueAttribute][
'attribute_list'].append(
(valueAttributeCount, keyAttribute))
CommonUtilities.writeDictToJson(self.collision_inv_dict, self.outFile)
self.exc_end_time = datetime.datetime.now()
def smartReturn(res, cols): """ 根据查询的字段数，如果只有一个字段，直接将结果存入一个列表中返回，否则返回一个键值对组成的列表 :param res:查询结果序列 :param cols:查询字段序列 :return: """ if len(cols) == 1: return [ row[0] for row in res ] else: return CommonUtilities.translateQueryResIntoDict(cols, res)
def __getSimilarAtrr(self):
patterAtrr = []
##Elenco tutte le chiavi del file
keys = list(self.__current_json_data.keys())
###Per Ogni Chiave cerco chiavi da accoppiare nel dictSIM dinamico
for x in range(0, len(keys)):
common_el = CommonUtilities.common_elements(keys, self.coll_sim_din[keys[x]])
if len(common_el) > 1:
patterAtrr.append((keys[x], common_el))
return len(patterAtrr) > 0, patterAtrr
def getUserInfoByIdOrName(IdOrName): """ 根据ID或者工号模糊查询获取对应的员工信息 :param IdOrName: ID或者员工名称的信息 :return: """ raw = Raw_sql() raw.sql = """SELECT ID, Name, Password, DepartmentID, JobID, Permission, CONVERT(varchar(16), CreateTime, 20) CreateTime, CONVERT(varchar(16), LastModifiedTime, 20) LastModifiedTime FROM RMI_ACCOUNT_USER WHERE ID LIKE '%%%%%s%%%%' OR Name LIKE '%%%%%s%%%%'"""%(IdOrName, IdOrName) res, cols = raw.query_all(needColumnName=True) return CommonUtilities.translateQueryResIntoDict(res=res, columns=cols)
def getPagedInfo(self, pageNo, pageSize, columns, columnsAlternativeNames, whereColumns, whereValues, orderString):
"""
分页查询接口
:param pageNo:页码
:param pageSize:页面大小
:param columns:字段
:param columnsAlternativeNames:字段替代名
:param whereColumns:where语句的字段列表
:param whereValues:where语句的字段列表对应的值
:param orderString:排序语句
:return:{''}
"""
res, cols, count = self.raw.pagedQuery(pageNo, pageSize, self.dataSourceTable, self.primaryKey, self.formatColumnString(columns, columnsAlternativeNames),
self.formatFuzzyWhereString(whereColumns, whereValues), orderString, needCounts=True, needColumnName=True )
return {'listData':CommonUtilities.translateQueryResIntoDict(cols, res), 'count':count}
def getAllMaterialByName(fuzzyName):
"""
根据模糊输入获取所有材料的名称
:param fuzzyName:模糊输入
:return:{'id':材料名称ID,'name':材料名称,'cata':材料种类名称}
"""
raw = rawSql.Raw_sql()
raw.sql = """SELECT MaterialID AS id, MaterialName AS name, dbo.getMaterialTypeNameByID(MaterialTypeID) AS cata
FROM RMI_MATERIAL_NAME WITH(NOLOCK)"""
if fuzzyName:
raw.sql += """ WHERE MaterialName LIKE '%%%%%s%%%%'"""%fuzzyName
res, cols = raw.query_all(needColumnName=True)
return CommonUtilities.translateQueryResIntoDict(cols, res)
else: #如果为空返回空数据，否则前端卡顿
return [{"name":u'请输入关键字', "id":"", "cata":""}]
def __DynColSim(self):
###Unisco Prima sulla base dell'uguaglianza dei valori
for AttrName, AttrValues in self.jsData.items():
self.dyn_col_sim[AttrName] = { }
AttrValues_Set = set([AttrValue for AttrPath, AttrValue in AttrValues])
for AttrValue in AttrValues_Set:
###Per ogmi valore Attributo di AttrName cerco valori Attr uguali e simili
similarValueList = CommonUtilities.getSimilarValues(AttrValue, self.col_inv.keys())
#print(AttrValue, similarValueList)
###Per ogni valore Attributo Trovato cerco a quale chiave è associato e lo aggiungo
for similarValue in similarValueList:
keysOfAttributes = self.col_inv[similarValue]
for key, key_count in keysOfAttributes.items():
if not key in self.dyn_col_sim[AttrName].keys():
self.dyn_col_sim[AttrName][key] = 0
self.dyn_col_sim[AttrName][key] += key_count
def getSuppliersAssessmentDataByDate(start, end):
"""
根据时间跨度范围来获取供应商评审数据
:param start:开始时间 e.g. 2016-01-01
:param end:结束时间 e.g. 2016-02-02
:return:[{"GongYingShangMingCheng": 供应商名称, "BuHeGeShuLiang": 统计期内不合格批次数量,
"TongJiQiNeiDaoHuoPiCi": 统计期内到货批次, "GongHuoShuLiang": 供货数量, "GongYingShangBianMa": 供应商编码,
"JinHuoJianYanHeGeLv": 进货检验合格率, "BuHeGePiCi": 不合格批次数量},...]
"""
raw = rawSql.Raw_sql()
start = "-".join([number.rjust(2,'0') if len(number) < 2 else number for number in start.split('-')])
end = "-".join([number.rjust(2,'0') if len(number) < 2 else number for number in end.split('-')])
raw.sql = """SELECT GongYingShangBianMa, GongYingShangMingCheng,
CAST(SUM(GongHuoShuLiang) as varchar(50)) + DaoHuoShuLiangDanWei GongHuoShuLiang, SUM(TongJiQiNeiDaoHuoPiCi) TongJiQiNeiDaoHuoPiCi,
SUM(BuHeGePiCi) BuHeGePiCi, SUM(BuHeGeShuLiang) BuHeGeShuLiang,
CAST(CAST( 1 - CAST(SUM(BuHeGePiCi) AS DECIMAL(9,2)) / CAST(SUM(TongJiQiNeiDaoHuoPiCi) AS DECIMAL(9,2)) AS DECIMAL(9,2)) * 100 AS INT) JinHuoJianYanHeGeLv
FROM dbo.SupplierInfoAnalysis
WHERE RiQi >= '%s' AND RiQi <= '%s'
GROUP BY GongYingShangBianMa, GongYingShangMingCheng, DaoHuoShuLiangDanWei"""%(start,end)
res, cols = raw.query_all(needColumnName=True)
return CommonUtilities.translateQueryResIntoDict(columns=cols, res=res)
def cleanKeys(self):
uuidKey = 0
for key, value in self.srcData.items():
if key == "<page title>":
continue
strkCleaner = StringKeyCleaner(key)
newKey = strkCleaner.cleanKeyStr()
if not newKey in self.validKeys:
continue
if type(value).__name__ == 'list':
value = CommonUtilities.listToStringOrSetList(value)
if not newKey == key and newKey in self.dstData.keys():
newKey += str(uuidKey)
uuidKey +=1
if len(newKey) and not type(value).__name__ == 'list':
self.dstData[newKey] = value
elif type(value).__name__ == 'list':
value_cl = StringCompositeValueCleaner(value)
value = value_cl.cleanValues()
self.compositeValues[key] = value
else:
self.droppedKeys[key] = value
def __AggregateFileSameSourceAndSpect(self):
progressCount = 0
print("")
CommonUtilities.progressBar(progressCount, len(self.lk_path_dict.keys()), status="Loading ..")
for objectSpect, objectSources in self.lk_path_dict.items():
progressCount +=1
CommonUtilities.progressBar(progressCount, len(self.lk_path_dict.keys()), status=f"Agg: {objectSpect}")
self.lk_1_path_dict[objectSpect] = {}
for sources, filespath in objectSources.items():
self.__findOrCreateDir(sources)
self.lk_1_path_dict[objectSpect][sources] = f"{sources}/{self.newFileNameID}"
new_merged_file = self.__AggregateFiles(filespath)
CommonUtilities.writeDictToJson(new_merged_file, f"{self.dst_dir_name}/{sources}/{self.newFileNameID}.json")
self.newFileNameID += 1
def __getAttrRelevance(self, attrName, attrValue):
#Prendo tutti i nomi attributi correlati a quel valore
keysMatchingValue = self.__coll_inv_dict[attrValue]['attribute_list']
#Prendo tutti i valori associati al nome attributo
valuesOfAttribute = self.__coll_dict[attrName]['value_list']
##Calcolo valore e totale di ogni insieme
key_count, ktotal_count = CommonUtilities.get_count_and_total_of(attrName, keysMatchingValue)
val_count, vtotal_count = CommonUtilities.get_count_and_total_of(attrValue, valuesOfAttribute)
###Per ogni insieme keysMatchingValue, valuesOfAttribute calcolo la media dei punteggi e il valor medio
keysMatchingValue_med = ktotal_count / len(keysMatchingValue)
keysMatchingValue_vmed = CommonUtilities.get_vMed(keysMatchingValue)
valuesOfAttribute_med = vtotal_count / len(valuesOfAttribute)
valuesOfAttribute_vmed = CommonUtilities.get_vMed(valuesOfAttribute)
######Calcolo il peso del nomeAttr/valorAttr rispetto all'insieme di appartenenza
prcAttrVInAttrName = float(key_count/ktotal_count)
prcAttrNameInAttrV = float(val_count/vtotal_count)
##Per keysMatchingValue, valuesOfAttribute scelgo il min tra media e valor medio
keysMatchingValue_min = float(min(keysMatchingValue_med, keysMatchingValue_vmed) / ktotal_count)
valuesOfAttribute_min = float(min(valuesOfAttribute_med, valuesOfAttribute_vmed) / vtotal_count)
if CommonUtilities.get_max_in_tuple_list(keysMatchingValue) == keysMatchingValue_min:
keysMatchingValue_min *= 0.9
if CommonUtilities.get_max_in_tuple_list(valuesOfAttribute) == valuesOfAttribute_min:
valuesOfAttribute_min *= 0.9
relevantScore = max( 0, (prcAttrVInAttrName - keysMatchingValue_min) , (prcAttrNameInAttrV - valuesOfAttribute_min) , (prcAttrVInAttrName - keysMatchingValue_min) + (prcAttrNameInAttrV - valuesOfAttribute_min) )
# if attrName == "ean13":
# print(attrName, attrValue)
# print(key_count, ktotal_count, prcAttrVInAttrName, keysMatchingValue_min, len(keysMatchingValue))
# print(val_count, vtotal_count, prcAttrNameInAttrV, valuesOfAttribute_min, len(valuesOfAttribute))
# print(f"Final Score: {relevantScore}")
return keysMatchingValue, float(relevantScore) * 100
def __AggregateFileSameSpect(self):
progressCount = 0
print("")
CommonUtilities.progressBar(progressCount,
len(self.lk_path_dict.keys()),
status="Loading ..")
for objectSpect, objectSources in self.lk_path_dict.items():
progressCount += 1
CommonUtilities.progressBar(progressCount,
len(self.lk_path_dict.keys()),
status=f"Agg: {objectSpect}")
pathToMerge = []
for sources, filespath in objectSources.items():
pathToMerge.append(filespath)
self.lk_2_path_dict[objectSpect] = f"{self.newFileNameID}"
new_merged_file = self.__AggregateFiles(pathToMerge)
CommonUtilities.writeDictToJson(
new_merged_file,
f"{self.dst_dir_name}/{self.newFileNameID}.json")
self.newFileNameID += 1
import CommonUtilities
from Constats_App import *
from fuzzywuzzy import fuzz
bigcluster = CommonUtilities.loadJsonFile(f"{PHASE_3_SOURCE_DIR}/big_cluster2")
keySimInv = CommonUtilities.loadJsonFile(f"{PHASE_3_SOURCE_DIR}/testInv")
outputData = {}
outputData2 = {}
outputData3 = {}
outputData4 = {}
outputData5 = {}
##Passo 1 count gli elementi per ogni nomeAttributo
for key, values in bigcluster.items():
if len(values) > 0:
outputData[key] = len(values)
CommonUtilities.writeDictToJson(outputData,
f"{PHASE_3_SOURCE_DIR}/big_clusterkey.json")
####Passo 2 Conto gli elementi di ogni chiave suddividendoli secondo il nome attributo originale
for key, values in bigcluster.items():
if len(values) > 0:
outputData2[key] = {}
for src, val, *oldAttrname in values:
if len(oldAttrname) > 0:
if not oldAttrname[0] in outputData2[key].keys():
outputData2[key][oldAttrname[0]] = 0
outputData2[key][oldAttrname[0]] += 1
else:
import CommonUtilities
from Constats_App import *
dictBigCluster2 = CommonUtilities.loadJsonFile(f"{PHASE_3_SOURCE_DIR}/big_cluster2.json", ext="")
clusterTaDict = CommonUtilities.loadJsonFile(f"{PHASE_3_SOURCE_DIR}/big_clusterkey_5.json", ext="")
clusterTaDictinv = {}
bigCluster3 = {}
for key, values in clusterTaDict.items():
for value in values:
clusterTaDictinv[value] = key
CommonUtilities.writeDictToJson(clusterTaDictinv, f"{PHASE_3_SOURCE_DIR}/big_clusterkey_5_inv.json")
for key, values in dictBigCluster2.items():
if len(values) > 0:
rootKey = clusterTaDictinv[key]
if not rootKey in bigCluster3.keys():
bigCluster3[rootKey] = []
for src, value, *oldAttrName in values:
if len(oldAttrName) < 1:
curr_item = (src, value, key)
else:
curr_item = (src, value, oldAttrName[0])
bigCluster3[rootKey].append(curr_item)
CommonUtilities.writeDictToJson(bigCluster3, f"{PHASE_3_SOURCE_DIR}/big_cluster3.json")
class CrudOperations:
# Creating object of CommonUtilities class
commonUtils = CommonUtilities()
product_collection = None
"""
Read the values from property files and store it to the respective variables
"""
sDbName = commonUtils.getDbDetails('DB_DETAILS', 'db.name')
sDbUsername = commonUtils.getDbDetails('DB_DETAILS', 'db.username')
sDbPassword = commonUtils.getDbDetails('DB_DETAILS', 'db.password')
sCollectionName = commonUtils.getDbDetails('DB_DETAILS',
'db.collection.name')
"""
This piece of code will run when the Object of the class is created.
If no errors in reading values from Properties file then connect to the Mongo DB cloud else set the variable value
"product_collection" to None.
On successfull connection to Mongo DB cloud, we get the collection Object in return
"""
if (sDbName is not None) and (sDbUsername is not None) and (
sDbPassword is not None) and (sCollectionName is not None):
sConnectionString = "mongodb://{0}:{1}@productcluster-shard-00-00.jcvab.mongodb.net:27017," \
"productcluster-shard-00-01.jcvab.mongodb.net:27017," \
"productcluster-shard-00-02.jcvab.mongodb.net:27017/" \
"{2}?ssl=true&replicaSet=atlas-14aexf-shard-0&authSource=admin" \
"&retryWrites=true&w=majority".format(sDbUsername, sDbPassword, sDbName)
connectMongoDbCloud = commonUtils.connectMongoDbCloud(
sConnectionString=sConnectionString,
sDbName=sDbName,
sCollectionName=sCollectionName)
if connectMongoDbCloud is not None:
product_collection = connectMongoDbCloud
else:
product_collection = None
else:
product_collection = None
print("FAILED to read the configuration data.")
def getAllProducts(self):
"""
This method is used to get all the documents from the connected product_collection object.
:return: After getting documents from collection, It returns the entire document in json format.
"""
if request.method == 'GET':
if self.product_collection is not None:
all_docs = self.product_collection.find({})
if all_docs.count() > 0:
# print("Inside doc", self.product_collection.count_documents({}))
return Response(json.dumps(list(all_docs), default=str),
mimetype="application/json"), 200
else:
return jsonify(success="false",
message="No data found"), 404
else:
return jsonify(success="false",
message="Unable to connect mongo DB :("), 500
else:
return jsonify(success="false", message="Method not allowed."), 405
def insertProduct(self):
"""
This method helps in inserting the data into the database
:return: Returns id of newly created object
"""
if request.method == 'POST':
if self.product_collection is not None:
_id = ObjectId()
try:
new_product = {
"_id":
_id,
"brand_name":
request.json['brand_name'],
"classification_l1":
request.json['classification_l1'],
"classification_l2":
request.json['classification_l2'],
"classification_l3":
request.json['classification_l3'],
"classification_l4":
request.json['classification_l4'],
"currency":
request.json['currency'],
"image_url":
request.json['image_url'],
"name":
request.json['name'],
"offer_price_value":
int(request.json['offer_price_value']),
"regular_price_value":
int(request.json['regular_price_value'])
}
# print(new_product)
self.product_collection.insert_one(new_product)
return jsonify(success=True,
message="Successfully inserted with id :" +
str(_id)), 201
except:
return jsonify(success="false",
message="Someting went wrong."), 500
else:
return jsonify(success="false",
message="Unable to connect mongo DB :("), 500
else:
return jsonify(success="false", message="Method not allowed."), 405
def operationOnSpecificProduct(self, idProduct):
"""
This method just calls respective method based on the requested method
:param idProduct: ID of the product on which we need to manipulate
:return: Incase of GET method - It return the object of specified ID, Incase of PUT and DELETE method - Just return success if manipulated successfully
"""
if request.method == 'GET':
return self.getSingleProduct(idProduct)
elif request.method == 'PUT':
return self.updateSingleProduct(idProduct)
elif request.method == 'DELETE':
return self.deleteSingleProduct(idProduct)
else:
return jsonify(success="false", message="Method not allowed."), 405
def getSingleProduct(self, idProduct):
"""
This method find the product in the DB by id
:param idProduct: ID of the product for which we need to send the detail of
:return: return object of specified ID if found else No data found
"""
try:
sProduct = self.product_collection.find(
{"_id": ObjectId(idProduct)})
if sProduct.count() > 0:
return Response(json.dumps(list(sProduct), default=str),
mimetype="application/json"), 200
else:
return jsonify(success="false", message="No data found")
except:
return jsonify(success="false",
message="Something went wrong."), 500
def updateSingleProduct(self, idProduct):
"""
This method is used to update all the fields of particular object, first it finds the object in the DB
if found then it updates it else No data found
:param idProduct: id of the product on which we need to manipulate
:return: returns success json if updated successfull else "Something went wrong" 500
"""
try:
sProduct = self.product_collection.find(
{"_id": ObjectId(idProduct)})
if sProduct.count() == 1:
self.product_collection.update({"_id": ObjectId(idProduct)}, {
"$set": {
"brand_name":
request.json['brand_name'],
"classification_l1":
request.json['classification_l1'],
"classification_l2":
request.json['classification_l2'],
"classification_l3":
request.json['classification_l3'],
"classification_l4":
request.json['classification_l4'],
"currency":
request.json['currency'],
"image_url":
request.json['image_url'],
"name":
request.json['name'],
"offer_price_value":
int(request.json['offer_price_value']),
"regular_price_value":
int(request.json['regular_price_value'])
}
})
return jsonify(success=True,
message="Successfully updated."), 200
else:
return jsonify(success="false", message="No data found."), 404
except:
return jsonify(success="false",
message="Something went wrong."), 500
def deleteSingleProduct(self, idProduct):
"""
This method is used to delete entire object from DB. First it will find the object in DB by ID
if found then delete it else return No data found.
:param idProduct: id of the product which we want to delete
:return: return success json if deleted successfully.
"""
try:
sProduct = self.product_collection.find(
{"_id": ObjectId(idProduct)})
if sProduct.count() == 1:
self.product_collection.delete_one(
{"_id": ObjectId(idProduct)})
return jsonify(success=True,
message="Successfully deleted."), 200
else:
return jsonify(success="false", message="No data found."), 404
except:
return jsonify(success="false",
message="Something went wrong."), 500
from Constats_App import *
import CommonUtilities
ordered_value = []
collisionInvData = CommonUtilities.loadJsonFile(COLLISION_DICTIONARY_INV_DICT,
ext="")
for key, val in collisionInvData.items():
ordered_value.append((key, len(val['attribute_list'])))
ordered_value.sort(key=lambda x: x[1], reverse=True)
for x in range(0, 50):
print(ordered_value[x])
import CommonUtilities
from Constats_App import *
dictBigCluster = CommonUtilities.loadJsonFile(
f"{PHASE_3_SOURCE_DIR}/big_cluster.json", ext="")
dictColl = {}
dictColl2 = {}
dictCollInv = {}
for key, values in dictBigCluster.items():
dictColl[key] = {}
for src, val, *out in values:
if not val in dictColl[key].keys():
dictColl[key][val] = 0
dictColl[key][val] += 1
CommonUtilities.writeDictToJson(dictColl,
f"{PHASE_3_SOURCE_DIR}/big_clusterColl.json")
for key, values in dictBigCluster.items():
dictColl2[key] = {}
for src, val, *out in values:
if not val in dictColl2[key].keys():
dictColl2[key][val] = {}
if len(out) > 0:
oldAttrName = out[0]
else:
oldAttrName = key
if not oldAttrName in dictColl2[key][val].keys():
def LoadPath(self):
self.lk_2_path_dict = CommonUtilities.loadJsonFile(
self.dst_lk_path_dict, ext='')
from Constats_App import *
import CommonUtilities
dimdict_1 = CommonUtilities.loadJsonFile(COLLISION_DICTIONARY_SIM_DYN_DICT_01,
ext="")
dimdict_2 = CommonUtilities.loadJsonFile(COLLISION_DICTIONARY_SIM_DYN_DICT_02,
ext="")
set_1 = set(dimdict_1.keys())
set_2 = set(dimdict_2.keys())
print(set_2.symmetric_difference(set_1))
print(len(set_1), len(set_2), len(set_2.intersection(set_1)),
len(set_2.symmetric_difference(set_1)))
import pandas as pd
from tqdm import tqdm
import json
from collections import defaultdict
import itertools
import CommonUtilities
# Leggo la ground truth in un Pandas dataframe
pos_df = pd.read_csv('sources_3/gt_onevalue.csv')
attrNameSet = []
# ordino lessicograficamente la GT
for index, row in pos_df.iterrows():
left_ia = row['left_instance_attribute']
right_ia = row['right_instance_attribute']
attrNameSet.append(left_ia)
attrNameSet.append(right_ia)
CommonUtilities.progressBar(index + 1, pos_df.shape[0],
f"{index+1}/{pos_df.shape[0]}")
attrNameSet = set(attrNameSet)
with open("attrToAnalizeFull.txt", "w") as attrFile:
for attr in sorted(list(attrNameSet)):
attrFile.write(f"{attr}\n")
import CommonUtilities
with open("attrToAnalizeFull.txt", "r") as AttrNamesF:
gtAttrNames = AttrNamesF.readlines()
gtAttrNames = set([line[:-1] for line in gtAttrNames])
with open(f"sources_3/big_cluster3_refactor.json", 'r') as f:
distros_dict = json.load(f)
newLine = ''
arrResult = ["left_instance_attribute,right_instance_attribute\n"]
for key, value in distros_dict.items():
###Faccio il prodotto cartesiano e ordino le coppie
print(f"Working On {key}")
current_list = set(value)
slist = sorted(list(current_list.intersection(gtAttrNames)))
elementCount = len(slist)
print("List sorted")
for x in range(0, elementCount):
currente = slist[x]
ll = [currente + "," + s + "\n" for s in slist[x + 1:]]
arrResult += ll
CommonUtilities.progressBar(x + 1, elementCount,
f"{x}/{elementCount}")
print(f"Start Writing: {len(arrResult)} lines of {key}")
with open(f"sources_3/custom_ground.csv", "a+") as gF:
gF.writelines(arrResult)
arrResult = []
print(f"File Saved")
def __loadCollisionDictionary(self):
self.collision_dict = CommonUtilities.loadJsonFile(self.outFile,
ext="")
print(f"{type(self).__name__} CollisionDictionary Loaded!")