def main():
# Stores the execution start time to calculate the time it takes for the module to execute.
initial_time = time.time()
# Define el logger que se utilizará.
logger = log4p.GetLogger(__name__)
log = logger.logger
env_path = Path(ROOT_DIR) / 'config' / (SYN_ENV + '.env')
load_dotenv(dotenv_path=env_path)
# Check if there is a running process that contains the name of this module.
check_same_python_module_already_running(os.path.split(__file__))
# Establece los parámetros que se enviarán en la petición.
mongodb_client: MongoClient = get_default_mongo_client()
db = mongodb_client[os.environ['JIRA_MONGODB_DATABASE_NAME']]
log.debug(f"Colecciones exitentes en '{db.name}': {str(db.list_collection_names())}")
for project in os.environ["JIRA_PROJECT_NAME"].split(","):
for year in range(2001, 2021):
col = db[f"{project.lower()}_{year}_{year + 1}"]
if col.name in db.list_collection_names():
db.drop_collection(col.name)
final_time = time.time()
log.info(f"Total execution time = {((final_time - initial_time) / 60)} minutos")
def main():
# Stores the execution start time to calculate the time it takes for the module to execute.
initial_time = time.time()
# Define el logger que se utilizará.
logger = log4p.GetLogger(__name__)
log = logger.logger
env_path = Path(ROOT_DIR) / 'config' / (SYN_ENV + '.env')
load_dotenv(dotenv_path=env_path)
# Check if there is a running process that contains the name of this module.
check_same_python_module_already_running(os.path.split(__file__))
for project in os.environ["BUGZILLA_PROJECT_NAME"].split(","):
for year in range(int(os.environ['BUGZILLA_FIRST_CREATION_YEAR']),
int(os.environ['BUGZILLA_LAST_CREATION_YEAR'])):
cmd = f"python CreateBugzillaMongoDBCollection.py --p {project}" \
f" --db {os.environ['BUGZILLA_MONGODB_DATABASE_NAME']}" \
f" --c {project.lower()} --y {year} --sm 1 --em 12"
# Ejecuta el comando en la consola Windows.
check_output(cmd, shell=True)
print(cmd)
final_time = time.time()
log.info(
f"Total execution time = {((final_time - initial_time) / 60)} minutos")
def get_attention_vector_raw_data(db_name,
col_name,
categorical=True,
column=None):
# Stores the execution start time to calculate the time it takes for the module to execute.
initial_time = time.time()
# Define el logger que se utilizará.
logger = log4p.GetLogger(__name__)
log = logger.logger
log.debug(f"\n[INICIO EJECUCIÓN]")
env_path = Path(ROOT_DIR) / 'config' / (SYN_ENV + '.env')
load_dotenv(dotenv_path=env_path)
mongodb_client: MongoClient = get_default_mongo_client() if os.environ['WORK_ENVIRONMENT'] == 'aws' \
else get_default_local_mongo_client()
db = mongodb_client[db_name]
col = db[col_name]
if not categorical:
query, fields = get_raw_data_query_and_projection()
fields["constituents_embeddings_description1"] = 1
fields["constituents_embeddings_description2"] = 1
query["constituents_embeddings_description1.0"] = {'$exists': True}
query["constituents_embeddings_description2.0"] = {'$exists': True}
else:
query, fields = get_raw_data_query_and_projection_categorical(
column=column)
fields["constituents_embeddings"] = 1
query["constituents_embeddings.0"] = {'$exists': True}
return col.find(query, fields)
def main():
# Stores the execution start time to calculate the time it takes for the module to execute.
initial_time = time.time()
# Define el logger que se utilizará.
logger = log4p.GetLogger(__name__)
log = logger.logger
# Check if there is a running process that contains the name of this module.
check_same_python_module_already_running(os.path.split(__file__))
# Carga el fichero de configuración para el entorno.
env_path = Path(ROOT_DIR) / 'config' / (SYN_ENV + '.env')
load_dotenv(dotenv_path=env_path)
# Inicializa los parámetros MongoDB para almacenar las estadísticas.
mongodb_client: MongoClient = get_default_mongo_client()
db = mongodb_client[os.environ["JIRA_MONGODB_DATABASE_NAME"]]
col = db[os.environ["JIRA_STATISTICS_MONGODB_COLLECTION_NAME"]]
# Inicializa el resultado.
statistics_dict = {}
for project in os.environ["JIRA_PROJECT_NAME"].split(","):
statistics_dict["project"] = project
total_bugs = 0
statistics_json = {}
for year in range(int(os.environ["JIRA_FIRST_CREATION_YEAR"]), datetime.datetime.now().year + 1):
statistics_dict[year] = {}
statistics_dict[year]["_total"] = 0
total_bugs_year = 0
# month_statistics_dict = {}
for month in range(1, 13):
max_year = year
max_month = month + 1
if max_month > 12:
max_month = 1
max_year += 1
issues = get_issues_by_date_range(
project=project,
min_created_date=f"{year}-{str(month).zfill(2)}-01",
max_creation_date=f"{max_year}-{str(max_month).zfill(2)}-01",
max_results=-False,
fields="id"
)
# Número total de incidencias del mes analizado.
# month_statistics_dict.append({"month": month, "count": len(bugs)})
# month_statistics_dict.append({month: len(bugs)})
statistics_dict[year][month] = len(issues)
total_bugs_year += len(issues)
# Número total de incidencias del año analizado.
statistics_dict[year]["_total"] = total_bugs_year
# Estadísticas mensuales.
# statistics_dict[year]["bugs"] = month_statistics_dict
log.info(json.dumps(statistics_dict))
# Número total de incidencias del proyecto.
total_bugs += total_bugs_year
statistics_dict["_total"] = total_bugs
# Almacena las estadísticas para el proyecto.
statistics_json = json.dumps(statistics_dict)
col.insert_one(json.loads(statistics_json))
def main():
# Stores the execution start time to calculate the time it takes for the module to execute.
initial_time = time.time()
# Define el logger que se utilizará.
logger = log4p.GetLogger(__name__)
log = logger.logger
env_path = Path(ROOT_DIR) / 'config' / (SYN_ENV + '.env')
load_dotenv(dotenv_path=env_path)
# Check if there is a running process that contains the name of this module.
check_same_python_module_already_running(os.path.split(__file__))
for project in os.environ["JIRA_PROJECT_NAME"].split(","):
for year in range(2001, 2021):
cmd = f"mongoimport /host {os.environ['MONGO_HOST_IP']} /port {os.environ['MONGO_PORT']}" \
f" /username {os.environ['MONGO_USERNAME']} /password {os.environ['MONGO_PASSWORD']}" \
f" /authenticationDatabase admin /authenticationMechanism SCRAM-SHA-1" \
f" /db {os.environ['JIRA_MONGODB_DATABASE_NAME']}" \
f" /collection {project.lower()}_all" \
f" /file {Path(ROOT_DIR) / 'data' / project.lower()}_{year}_{year + 1}.json"
# Ejecuta el comando en la consola Windows.
check_output(cmd, shell=True)
print(cmd)
final_time = time.time()
log.info(
f"Total execution time = {((final_time - initial_time) / 60)} minutos")
class OpenApiManager:
__logger = log4p.GetLogger(__name__, logging_level=settings.DEBUG_LEVEL, config=settings.LOGGING_CONFIG)
__data_path = os.path.dirname(settings.ROOT_DIR) + '/' + settings.DATA_DIR_ZIPCODE
__query_column = ['시도', '시군구', '읍면', '건물번호본번', '건물번호부번']
__kakao_Apis = KakaoOpenAPIs()
__estate_Apis = EstateOpenAPIs()
__kakao_restApiKey = keys.KeyManager.getKakaoKeys()
__gov_restApiKey = keys.KeyManager.getGovKeys()
def __init__(self):
self.result = 0
def getMapAdderess(self, query):
return json.loads(self.__kakao_Apis.getAddress(self.__kakao_Apis[0], query))
def makeQuery(self, sido):
return list(map(lambda x: '{} {} {} {}-{}'.format(x[0], x[1], x[2], x[3], x[4])
, self.getZipcode(sido)[self.__query_column].values))
def getFileName(self):
return list(filter(lambda x: x.endswith('.txt'), os.listdir(self.__data_path)))
def getZipcode(self, sido):
return pd.read_csv(self.__data_path + sido, delimiter='|')
def xmlToJson(self, xml):
return json.dumps(xmltodict.parse(xml), indent=4)
def totalColumnCount(self):
cnt = 0
for i in self.getFileName():
cnt += len(self.getZipcode(i))
# 6,319,248
print(cnt)
def main():
# Stores the execution start time to calculate the time it takes for the module to execute.
initial_time = time.time()
# Define el logger que se utilizará.
logger = log4p.GetLogger(__name__)
log = logger.logger
log.info(f"INICIO DE LA EJECUCIÓN")
env_path = Path(ROOT_DIR) / 'config' / (SYN_ENV + '.env')
load_dotenv(dotenv_path=env_path)
# Check if there is a running process that contains the name of this module.
check_same_python_module_already_running(os.path.split(__file__))
# Incializa las variables que almacenarán los argumentos de entrada.
input_params = get_input_params()
dup = TreeLstmDuplicateTrain(corpus='bugzilla',
collection='clear',
attention=False,
attention_size=10,
glove_size=100,
hidden_size=100,
max_input=200,
batch_size=1,
optimizer='ADAM',
learning_rate=0.001,
update_embeddings=True,
patience=5).load_or_run()
output_dir = 'resultados/dump'
class TestDBAdder(unittest.TestCase):
__test_list = [{'documents': [{'address': {'address_name': '강원 강릉시 강동면 심곡리 20', 'b_code': '4215034029', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '20', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 심곡리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '129.051594222091', 'y': '37.6692864711714', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 심곡리 20', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '129.051594222091', 'y': '37.6692864711714'}, {'address': {'address_name': '강원 강릉시 강동면 임곡리 20', 'b_code': '4215034025', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '20', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 임곡리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.992092081113', 'y': '37.6969282757675', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 임곡리 20', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.992092081113', 'y': '37.6969282757675'}], 'meta': {'is_end': True, 'pageable_count': 2, 'total_count': 2}}, {'documents': [{'address': {'address_name': '강원 강릉시 강동면 심곡리 46', 'b_code': '4215034029', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '46', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 심곡리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '129.048329561257', 'y': '37.667611743421', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 심곡리 46', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '129.048329561257', 'y': '37.667611743421'}, {'address': {'address_name': '강원 강릉시 강동면 안인리 46', 'b_code': '4215034023', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '46', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 안인리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.981777324901', 'y': '37.7422067212921', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 안인리 46', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.981777324901', 'y': '37.7422067212921'}, {'address': {'address_name': '강원 강릉시 강동면 정동진리 46', 'b_code': '4215034028', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '46', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 정동진리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '129.039466540445', 'y': '37.685181832718', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 정동진리 46', 'address_type': 'REGION_ADDR', 'road_address': {'address_name': '강원 강릉시 강동면 헌화로 969', 'building_name': '', 'main_building_no': '969', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_name': '강동면 정동진리', 'road_name': '헌화로', 'sub_building_no': '', 'undergroun_yn': '', 'underground_yn': 'N', 'x': '129.039389654758', 'y': '37.685035786709', 'zone_no': '25631'}, 'x': '129.039466540445', 'y': '37.685181832718'}], 'meta': {'is_end': True, 'pageable_count': 3, 'total_count': 3}}, {'documents': [{'address': {'address_name': '강원 강릉시 강동면 상시동리 54', 'b_code': '4215034021', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '54', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 상시동리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.954256398315', 'y': '37.7221376536465', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 상시동리 54', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.954256398315', 'y': '37.7221376536465'}, {'address': {'address_name': '강원 강릉시 강동면 심곡리 54', 'b_code': '4215034029', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '54', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 심곡리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '129.04977339423', 'y': '37.6672148766874', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 심곡리 54', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '129.04977339423', 'y': '37.6672148766874'}, {'address': {'address_name': '강원 강릉시 강동면 언별리 54', 'b_code': '4215034031', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '54', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 언별리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.950991930149', 'y': '37.7064159505469', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 언별리 54', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.950991930149', 'y': '37.7064159505469'}, {'address': {'address_name': '강원 강릉시 강동면 임곡리 54', 'b_code': '4215034025', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '54', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 임곡리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.988546280882', 'y': '37.6959092139482', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 임곡리 54', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.988546280882', 'y': '37.6959092139482'}, {'address': {'address_name': '강원 강릉시 강동면 정동진리 54', 'b_code': '4215034028', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '54', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 정동진리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '129.039158295033', 'y': '37.6834875646506', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 정동진리 54', 'address_type': 'REGION_ADDR', 'road_address': {'address_name': '강원 강릉시 강동면 정동5길 18', 'building_name': '', 'main_building_no': '18', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_name': '강동면 정동진리', 'road_name': '정동5길', 'sub_building_no': '', 'undergroun_yn': '', 'underground_yn': 'N', 'x': '129.039174126002', 'y': '37.6834613468307', 'zone_no': '25631'}, 'x': '129.039158295033', 'y': '37.6834875646506'}, {'address': {'address_name': '강원 강릉시 강동면 하시동리 54', 'b_code': '4215034026', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '54', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 하시동리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.972403508162', 'y': '37.7373375364249', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 하시동리 54', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.972403508162', 'y': '37.7373375364249'}], 'meta': {'is_end': True, 'pageable_count': 6, 'total_count': 6}}, {'documents': [{'address': {'address_name': '강원 강릉시 강동면 상시동리 112-3', 'b_code': '4215034021', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '112', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 상시동리', 'sub_adderss_no': '', 'sub_address_no': '3', 'x': '128.950303467111', 'y': '37.7259454953002', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 상시동리 112-3', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.950303467111', 'y': '37.7259454953002'}, {'address': {'address_name': '강원 강릉시 강동면 하시동리 112-3', 'b_code': '4215034026', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '112', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 하시동리', 'sub_adderss_no': '', 'sub_address_no': '3', 'x': '128.970164645192', 'y': '37.7342851273995', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 하시동리 112-3', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.970164645192', 'y': '37.7342851273995'}], 'meta': {'is_end': True, 'pageable_count': 2, 'total_count': 2}}, {'documents': [{'address': {'address_name': '강원 강릉시 강동면 모전리 235', 'b_code': '4215034022', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '235', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 모전리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.961635569263', 'y': '37.7087039867304', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 모전리 235', 'address_type': 'REGION_ADDR', 'road_address': {'address_name': '강원 강릉시 강동면 아래장작골길 167', 'building_name': '', 'main_building_no': '167', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_name': '강동면 모전리', 'road_name': '아래장작골길', 'sub_building_no': '', 'undergroun_yn': '', 'underground_yn': 'N', 'x': '128.961705292411', 'y': '37.7085616646508', 'zone_no': '25627'}, 'x': '128.961635569263', 'y': '37.7087039867304'}, {'address': {'address_name': '강원 강릉시 강동면 산성우리 235', 'b_code': '4215034030', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '235', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 산성우리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '129.013305689429', 'y': '37.681950623521', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 산성우리 235', 'address_type': 'REGION_ADDR', 'road_address': {'address_name': '강원 강릉시 강동면 오이동길 48-10', 'building_name': '', 'main_building_no': '48', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_name': '강동면 산성우리', 'road_name': '오이동길', 'sub_building_no': '10', 'undergroun_yn': '', 'underground_yn': 'N', 'x': '129.013336191729', 'y': '37.6818756034241', 'zone_no': '25630'}, 'x': '129.013305689429', 'y': '37.681950623521'}, {'address': {'address_name': '강원 강릉시 강동면 상시동리 235', 'b_code': '4215034021', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '235', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 상시동리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.952304138907', 'y': '37.7336031852503', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 상시동리 235', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.952304138907', 'y': '37.7336031852503'}, {'address': {'address_name': '강원 강릉시 강동면 안인진리 235', 'b_code': '4215034024', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '235', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 안인진리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.982047218548', 'y': '37.7292565732355', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 안인진리 235', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.982047218548', 'y': '37.7292565732355'}, {'address': {'address_name': '강원 강릉시 강동면 언별리 235', 'b_code': '4215034031', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '235', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 언별리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.945955149791', 'y': '37.6960427691512', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 언별리 235', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.945955149791', 'y': '37.6960427691512'}, {'address': {'address_name': '강원 강릉시 강동면 임곡리 235', 'b_code': '4215034025', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '235', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 임곡리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.972677871284', 'y': '37.6870415910054', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 임곡리 235', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.972677871284', 'y': '37.6870415910054'}, {'address': {'address_name': '강원 강릉시 강동면 하시동리 235', 'b_code': '4215034026', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '235', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 하시동리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.961524057389', 'y': '37.7442731623497', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 하시동리 235', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.961524057389', 'y': '37.7442731623497'}], 'meta': {'is_end': True, 'pageable_count': 7, 'total_count': 7}}, {'documents': [{'address': {'address_name': '강원 강릉시 강동면 산성우리 167', 'b_code': '4215034030', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '167', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 산성우리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '129.012889969006', 'y': '37.6841155858322', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 산성우리 167', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '129.012889969006', 'y': '37.6841155858322'}, {'address': {'address_name': '강원 강릉시 강동면 상시동리 167', 'b_code': '4215034021', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '167', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 상시동리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.953795840943', 'y': '37.7278517437565', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 상시동리 167', 'address_type': 'REGION_ADDR', 'road_address': {'address_name': '강원 강릉시 강동면 율곡로 2210', 'building_name': '', 'main_building_no': '2210', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_name': '강동면 상시동리', 'road_name': '율곡로', 'sub_building_no': '', 'undergroun_yn': '', 'underground_yn': 'N', 'x': '128.953812542146', 'y': '37.7278614661603', 'zone_no': '25620'}, 'x': '128.953795840943', 'y': '37.7278517437565'}, {'address': {'address_name': '강원 강릉시 강동면 언별리 167', 'b_code': '4215034031', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '167', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 언별리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.94917902346', 'y': '37.6979029850337', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 언별리 167', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.94917902346', 'y': '37.6979029850337'}], 'meta': {'is_end': True, 'pageable_count': 3, 'total_count': 3}}, {'documents': [{'address': {'address_name': '강원 강릉시 강동면 정동진리 47-7', 'b_code': '4215034028', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '47', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 정동진리', 'sub_adderss_no': '', 'sub_address_no': '7', 'x': '129.039676251305', 'y': '37.6854387250587', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 정동진리 47-7', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '129.039676251305', 'y': '37.6854387250587'}], 'meta': {'is_end': True, 'pageable_count': 1, 'total_count': 1}}, {'documents': [{'address': {'address_name': '강원 강릉시 강동면 모전리 21', 'b_code': '4215034022', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '21', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 모전리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.988759440491', 'y': '37.7076267705161', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 모전리 21', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.988759440491', 'y': '37.7076267705161'}, {'address': {'address_name': '강원 강릉시 강동면 심곡리 21', 'b_code': '4215034029', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '21', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 심곡리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '129.051211155321', 'y': '37.6688083081899', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 심곡리 21', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '129.051211155321', 'y': '37.6688083081899'}, {'address': {'address_name': '강원 강릉시 강동면 안인진리 21', 'b_code': '4215034024', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '21', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 안인진리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.985835294237', 'y': '37.7379492571895', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 안인진리 21', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.985835294237', 'y': '37.7379492571895'}, {'address': {'address_name': '강원 강릉시 강동면 임곡리 21', 'b_code': '4215034025', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '21', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 임곡리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.990843217808', 'y': '37.695907916971', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 임곡리 21', 'address_type': 'REGION_ADDR', 'road_address': {'address_name': '강원 강릉시 강동면 임곡로 529-21', 'building_name': '', 'main_building_no': '529', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_name': '강동면 임곡리', 'road_name': '임곡로', 'sub_building_no': '21', 'undergroun_yn': '', 'underground_yn': 'N', 'x': '128.989304997762', 'y': '37.6953469908661', 'zone_no': '25629'}, 'x': '128.990843217808', 'y': '37.695907916971'}], 'meta': {'is_end': True, 'pageable_count': 4, 'total_count': 4}}, {'documents': [{'address': {'address_name': '강원 강릉시 강동면 산성우리 314', 'b_code': '4215034030', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '314', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 산성우리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '129.015397017955', 'y': '37.6602548801982', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 산성우리 314', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '129.015397017955', 'y': '37.6602548801982'}, {'address': {'address_name': '강원 강릉시 강동면 안인진리 314', 'b_code': '4215034024', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '314', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 안인진리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.98907160023', 'y': '37.735544037936', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 안인진리 314', 'address_type': 'REGION_ADDR', 'road_address': {'address_name': '강원 강릉시 강동면 안인진길 43', 'building_name': '', 'main_building_no': '43', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_name': '강동면 안인진리', 'road_name': '안인진길', 'sub_building_no': '', 'undergroun_yn': '', 'underground_yn': 'N', 'x': '128.989046785392', 'y': '37.7355238277541', 'zone_no': '25626'}, 'x': '128.98907160023', 'y': '37.735544037936'}, {'address': {'address_name': '강원 강릉시 강동면 언별리 314', 'b_code': '4215034031', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '314', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 언별리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.941517466144', 'y': '37.6892176351255', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 언별리 314', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.941517466144', 'y': '37.6892176351255'}, {'address': {'address_name': '강원 강릉시 강동면 임곡리 314', 'b_code': '4215034025', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '314', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 임곡리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.970067915393', 'y': '37.6801722640821', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 임곡리 314', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.970067915393', 'y': '37.6801722640821'}, {'address': {'address_name': '강원 강릉시 강동면 정동진리 314', 'b_code': '4215034028', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '314', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 정동진리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '129.032698943391', 'y': '37.6907852451725', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 정동진리 314', 'address_type': 'REGION_ADDR', 'road_address': {'address_name': '강원 강릉시 강동면 정동1길 7', 'building_name': '', 'main_building_no': '7', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_name': '강동면 정동진리', 'road_name': '정동1길', 'sub_building_no': '', 'undergroun_yn': '', 'underground_yn': 'N', 'x': '129.032700936407', 'y': '37.6907793553935', 'zone_no': '25631'}, 'x': '129.032698943391', 'y': '37.6907852451725'}, {'address': {'address_name': '강원 강릉시 강동면 하시동리 314', 'b_code': '4215034026', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '314', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 하시동리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.964683156596', 'y': '37.7391307194766', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 하시동리 314', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.964683156596', 'y': '37.7391307194766'}], 'meta': {'is_end': True, 'pageable_count': 6, 'total_count': 6}}, {'documents': [{'address': {'address_name': '강원 강릉시 강동면 산성우리 143', 'b_code': '4215034030', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '143', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 산성우리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '129.013234517149', 'y': '37.6826636798871', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 산성우리 143', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '129.013234517149', 'y': '37.6826636798871'}, {'address': {'address_name': '강원 강릉시 강동면 심곡리 143', 'b_code': '4215034029', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '143', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 심곡리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '129.051861521889', 'y': '37.6633350604641', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 심곡리 143', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '129.051861521889', 'y': '37.6633350604641'}, {'address': {'address_name': '강원 강릉시 강동면 안인리 143', 'b_code': '4215034023', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '143', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 안인리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.976430364018', 'y': '37.7370772827441', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 안인리 143', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.976430364018', 'y': '37.7370772827441'}, {'address': {'address_name': '강원 강릉시 강동면 언별리 143', 'b_code': '4215034031', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '143', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 언별리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.949716919996', 'y': '37.6969703645559', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 언별리 143', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.949716919996', 'y': '37.6969703645559'}, {'address': {'address_name': '강원 강릉시 강동면 임곡리 143', 'b_code': '4215034025', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '143', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 임곡리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.975750499816', 'y': '37.6911037325394', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 임곡리 143', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '128.975750499816', 'y': '37.6911037325394'}, {'address': {'address_name': '강원 강릉시 강동면 정동진리 143', 'b_code': '4215034028', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '143', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 정동진리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '129.034855366468', 'y': '37.686006507933', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 정동진리 143', 'address_type': 'REGION_ADDR', 'road_address': None, 'x': '129.034855366468', 'y': '37.686006507933'}, {'address': {'address_name': '강원 강릉시 강동면 하시동리 143', 'b_code': '4215034026', 'h_code': '4215034000', 'main_adderss_no': '', 'main_address_no': '143', 'mountain_yn': 'N', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_h_name': '강동면', 'region_3depth_name': '강동면 하시동리', 'sub_adderss_no': '', 'sub_address_no': '', 'x': '128.971898309626', 'y': '37.7368983671856', 'zip_code': ''}, 'address_name': '강원 강릉시 강동면 하시동리 143', 'address_type': 'REGION_ADDR', 'road_address': {'address_name': '강원 강릉시 강동면 원장봉길 36-20', 'building_name': '', 'main_building_no': '36', 'region_1depth_name': '강원', 'region_2depth_name': '강릉시', 'region_3depth_name': '강동면 하시동리', 'road_name': '원장봉길', 'sub_building_no': '20', 'undergroun_yn': '', 'underground_yn': 'N', 'x': '128.971886800986', 'y': '37.7368579321326', 'zone_no': '25620'}, 'x': '128.971898309626', 'y': '37.7368983671856'}], 'meta': {'is_end': True, 'pageable_count': 7, 'total_count': 7}}]
__logger = log4p.GetLogger(__name__, logging_level=settings.DEBUG_LEVEL, config=settings.LOGGING_CONFIG)
__adder = adder
__db = adder.DBAdder()
def test_store_to_file(self):
self.__logger.logger.debug(self.__adder.parallel_run(10, 10, self.__test_list))
self.__logger.logger.info(self.__adder.parallel_run(10, 10, self.__test_list)) # add history
def fun1():
SCRIPT_NAME = os.path.basename(__file__)
print(SCRIPT_NAME)
config = "log4p.json"
print(config)
pLogger = log4p.GetLogger(SCRIPT_NAME, logging.DEBUG, config).get_l()
pLogger.debug("Type some log.")
pLogger.error("error log")
pLogger.info("info log")
class EstateOpenAPIs:
__logger = log4p.GetLogger(__name__,
logging_level=settings.DEBUG_LEVEL,
config=settings.LOGGING_CONFIG)
__host = 'http://apis.data.go.kr'
def __init__(self):
self.result = 0
"""
def getCnrdlnService(self, address, page=1, size=10, fmt='json'):
api_name = '/1611000/nsdi/eios/CnrdlnService.{}?'.format(fmt)
params = urlencode({'query': address, 'page': page, 'size': size})
rest_api = self.__host + api_name + params
full_url = request.Request(rest_api, method='GET')
self.__logger.logger.debug("Full URL={}".format(full_url.get_full_url()))
self.__logger.logger.debug("query={}, page={}, size={}".format(address, page, size))
return request.urlopen(full_url).read().decode('utf-8')
"""
"""
표준지공시지가정보서비스
"""
def getReferLandPriceService(self,
restApiKey,
idcode,
stdryear,
page=1,
row=10,
fmt='json'):
api_name = '/1611000/nsdi/ReferLandPriceService/attr/getReferLandPriceAttr?ServiceKey={}&'.format(
restApiKey)
params = urlencode({
'ldCode': idcode,
'stdrYear': stdryear,
'format': fmt,
'numOfRows': row,
'pageNo': page
})
rest_api = self.__host + api_name + params
full_url = request.Request(rest_api, method='GET')
self.__logger.logger.debug(
"Estate API Host : {}, Private Key : {}".format(
self.__host, restApiKey))
self.__logger.logger.debug("Full URL={}".format(
full_url.get_full_url()))
self.__logger.logger.debug("idcode={}, page={}, row={}".format(
idcode, page, row))
return request.urlopen(full_url).read().decode('utf-8')
def main():
# Stores the execution start time to calculate the time it takes for the module to execute.
initial_time = time.time()
# Define el logger que se utilizará.
logger = log4p.GetLogger(__name__)
log = logger.logger
env_path = Path(ROOT_DIR) / 'config' / (SYN_ENV + '.env')
load_dotenv(dotenv_path=env_path)
# Check if there is a running process that contains the name of this module.
check_same_python_module_already_running(os.path.split(__file__))
# Incializa las variables que almacenarán los argumentos de entrada.
input_params = get_input_params()
# Establece los parámetros que se enviarán en la petición.
mongodb_client: MongoClient = get_default_mongo_client()
db = mongodb_client[input_params.db_name]
source_collection = db[input_params.collection_name]
target_collection = db[f"{input_params.collection_name}_embeddings"]
log.debug(
f"Colecciones exitentes en '{db.name}': {str(db.list_collection_names())}"
)
if target_collection.name in db.list_collection_names(
) and input_params.drop_collection:
db.drop_collection(target_collection.name)
cursor = source_collection.find({}, {
"creation_ts": "$creation_time",
"short_desc": "$summary",
"bug_status": "$status",
"bug_id": "$id",
"dup_id": "$dupe_of",
"resolution": 1,
"version": 1,
"product": 1,
"priority": 1,
"component": 1,
"delta_ts": 1,
"bug_severity": "$severity",
"description": "$comments.0",
"normalized_short_desc": 1,
"normalized_description": 1,
"comments": {
"$slice": 1
}
})
final_time = time.time()
log.info(
f"Total execution time = {((final_time - initial_time) / 60)} minutos")
def get_pretrained_embeddings(size=100, model='glove'):
# Stores the start time of the method execution to calculate the time it takes.
initial_time = time.time()
# Defines logger.
logger = log4p.GetLogger(__name__)
log = logger.logger
log.debug(f"\n[START OF EXECUTION]")
env_path = Path(ROOT_DIR) / 'config' / (SYN_ENV + '.env')
load_dotenv(dotenv_path=env_path)
# Vocabulary and pretrained word embeddings path.
pretrained_embeddings = {
'glove': {
'file':
Path(os.environ.get('WORD_EMBEDDINGS_PATH')) / model /
f"glove.6B.{size}d.txt",
'vocab':
Path(os.environ.get('WORD_EMBEDDINGS_PATH')) / model /
f"glove.vocab.6B.{size}d.txt",
'embeddings':
Path(os.environ.get('WORD_EMBEDDINGS_PATH')) / model /
f"glove.embeddings.6B.{size}d.npy"
}
}
pre_vocab = []
with open(pretrained_embeddings[model]['vocab'], encoding='utf-8') as fin:
log.info(
f"Reading pretrained word embeddings vocabulary: '{pretrained_embeddings[model]['vocab']}'."
)
for line in fin:
pre_vocab.append(cleanup(line.strip()))
# Reverse vocabulary.
pre_reverse_vocab: dict = {n: i for i, n in enumerate(pre_vocab)}
# Word embeddings.
log.info(
f"Reading pretrained word embeddings vocabulary: '{pretrained_embeddings[model]['embeddings']}'."
)
pre_embs: Union[Union[ndarray, Iterable, int, float, tuple, dict], Any] = \
np.load(pretrained_embeddings[model]['embeddings'])
log.info("Loaded pretrained word embeddings.")
log.debug(f"\n[END OF EXECUTION]")
final_time = time.time()
log.info(
f"Execution total time = {((final_time - initial_time) / 60)} minutes")
return PretrainedWordEmbedding(size=size,
reversed_vocabulary=pre_reverse_vocab,
embeddings=pre_embs)
def main():
# Stores the execution start time to calculate the time it takes for the module to execute.
initial_time = time.time()
# Define el logger que se utilizará.
logger = log4p.GetLogger(__name__)
log = logger.logger
env_path = Path(ROOT_DIR) / 'config' / (SYN_ENV + '.env')
load_dotenv(dotenv_path=env_path)
# Check if there is a running process that contains the name of this module.
check_same_python_module_already_running(os.path.split(__file__))
# Incializa las variables que almacenarán los argumentos de entrada.
input_params = get_input_params()
# Establece los parámetros que se enviarán en la petición.
mongodb_client: MongoClient = get_default_mongo_client()
db = mongodb_client[input_params['mongo_params'].db_name]
col = db[
f"{input_params['mongo_params'].collection_name}"
f"_{input_params['bz_api_params'].year}_{input_params['bz_api_params'].year + 1}"]
log.debug(
f"Colecciones exitentes en '{db.name}': {str(db.list_collection_names())}"
)
if col.name in db.list_collection_names(
) and input_params['mongo_params'].drop_collection:
db.drop_collection(col.name)
# Para cada año recupera las incidencias utilizando la API de Bugzilla.
max_year = input_params['bz_api_params'].year
for month in range(input_params['bz_api_params'].start_month,
input_params['bz_api_params'].end_month + 1):
max_month = month + 1
if max_month > 12:
max_month = 1
max_year += 1
bugs = get_bugzilla_bugs_by_date_range(
project=input_params['bz_api_params'].project,
min_creation_ts=
f"{input_params['bz_api_params'].year}-{str(month).zfill(2)}-01",
max_creation_ts=f"{max_year}-{str(max_month).zfill(2)}-01",
max_results=input_params['bz_api_params'].query_limit,
include_fields=input_params['bz_api_params'].include_fields,
get_comments=input_params['bz_api_params'].get_comments)
save_issues_to_mongodb(mongodb_collection=col, issues=bugs)
final_time = time.time()
log.info(
f"Total execution time = {((final_time - initial_time) / 60)} minutos")
def main():
# Stores the execution start time to calculate the time it takes for the module to execute.
initial_time = time.time()
# Define el logger que se utilizará.
logger = log4p.GetLogger(__name__)
log = logger.logger
# Check if there is a running process that contains the name of this module.
check_same_python_module_already_running(os.path.split(__file__))
# Carga el fichero de configuración para el entorno.
env_path = Path(ROOT_DIR) / 'config' / (SYN_ENV + '.env')
load_dotenv(dotenv_path=env_path)
# Inicializa los parámetros MongoDB para almacenar las estadísticas.
mongodb_client: MongoClient = get_default_mongo_client()
db = mongodb_client["bugzilla"]
eclipse_base_col = db["eclipse_base"]
# Campos seleccionados para los trabajos abordados en el proyecto de investigación.
fields = {"_id": 0}
eclipse_base_data = eclipse_base_col.find({}, fields)
# .limit(20)
# Expande el cursor y construye el DataFrame
eclipse_base = pd.DataFrame(list(eclipse_base_data))
eclipse_base_summary_col = db["clear_description_nlp_2"]
eclipse_base_summary_data = eclipse_base_summary_col.find({}, fields)
eclipse_base_summary = pd.DataFrame(list(eclipse_base_summary_data))
eclipse_base_data_description_col = db["clear_short_desc_nlp"]
eclipse_base_data_description_data = eclipse_base_data_description_col.find(
{}, fields)
eclipse_base_data_description = pd.DataFrame(
list(eclipse_base_data_description_data))
# Merge de los dataframes
eclipse_base_summary_description = pd.merge(eclipse_base_summary,
eclipse_base_data_description,
on='id')
eclipse_base_clear = pd.merge(eclipse_base,
eclipse_base_summary_description,
on='id')
# Almacena el dataframe en MongoDB.
db["eclipse_base_clear"].insert_many(eclipse_base_clear.to_dict('records'))
final_time = time.time()
log.info(
f"Total execution time = {((final_time - initial_time) / 60)} minutos")
class Caller(pykka.ThreadingActor):
__am = open_api_manager.OpenApiManager()
__logger = log4p.GetLogger(__name__,
logging_level=settings.DEBUG_LEVEL,
config=settings.LOGGING_CONFIG)
def map_address_call(self, query):
try:
json = self.__am.getMapAdderess(query)
self.__logger.logger.debug(f'Success: {json}')
return json
except Exception as e:
self.__logger.logger.debug(f'Failed resolving {query}')
return e
class KakaoOpenAPIs:
__logger = log4p.GetLogger(__name__, logging_level=settings.DEBUG_LEVEL, config=settings.LOGGING_CONFIG)
__host = 'https://dapi.kakao.com/v2/local/'
def __init__(self):
self.result = 0
def getAddress(self, restApiKey, address, page=1, size=10, fmt='json'):
api_name = 'search/address.{}?'.format(fmt)
params = urlencode({'query': address, 'page': page, 'size': size})
header = {'Authorization': 'KakaoAK ' + restApiKey}
rest_api = self.__host + api_name + params
full_url = request.Request(rest_api, headers=header, method='GET')
self.__logger.logger.debug("Kakao API Host : {}, Private Key : {}".format(self.__host, restApiKey))
self.__logger.logger.debug("Full URL={}".format(full_url.get_full_url()))
self.__logger.logger.debug("query={}, page={}, size={}".format(address, page, size))
return request.urlopen(full_url).read().decode('utf-8')
def parallel_run(pool_size, call_limit, queries):
__logger = log4p.GetLogger(__name__,
logging_level=settings.DEBUG_LEVEL,
config=settings.LOGGING_CONFIG)
callers = [Caller.start().proxy() for _ in range(pool_size)]
json_results = []
for i, query in enumerate(queries):
if i < call_limit:
json_results.append(callers[i %
len(callers)].map_address_call(query))
else:
pass
gathered_results = pykka.get_all(json_results)
pykka.ActorRegistry.stop_all()
__logger.logger.debug(gathered_results)
return gathered_results
def get_constituency_tree_raw_data(db_name, col_name, categorical=True):
# Stores the execution start time to calculate the time it takes for the module to execute.
initial_time = time.time()
# Define el logger que se utilizará.
logger = log4p.GetLogger(__name__)
log = logger.logger
log.debug(f"\n[INICIO EJECUCIÓN]")
env_path = Path(ROOT_DIR) / 'config' / (SYN_ENV + '.env')
load_dotenv(dotenv_path=env_path)
mongodb_client: MongoClient = get_default_mongo_client() if os.environ['WORK_ENVIRONMENT'] == 'aws' \
else get_default_local_mongo_client()
db = mongodb_client[db_name]
col = db[col_name]
if not categorical:
query, fields = get_raw_data_query_and_projection()
else:
query, fields = get_raw_data_query_and_projection_categorical()
return col.find(query, fields)
def get_vectorized_issue(db_name,
col_name,
embeddings_size=100,
attention_vector=True,
categorical=False,
column=None):
# Stores the start time of the method execution to calculate the time it takes.
initial_time = time.time()
# Defines logger.
logger = log4p.GetLogger(__name__)
log = logger.logger
log.debug(f"\n[START OF EXECUTION]")
env_path = Path(ROOT_DIR) / 'config' / (SYN_ENV + '.env')
load_dotenv(dotenv_path=env_path)
pretrained_embeddings = get_pretrained_embeddings(embeddings_size)
if not attention_vector:
vectorized_issue = VectorizedIssue(
pretrained_embeddings=pretrained_embeddings,
constituency_trees_raw_data=get_constituency_tree_raw_data(
db_name, col_name, categorical=categorical))
else:
vectorized_issue = VectorizedIssue(
pretrained_embeddings=pretrained_embeddings,
attention_vector_raw_data=get_attention_vector_raw_data(
db_name, col_name, categorical=categorical, column=column))
log.debug(f"\n[END OF EXECUTION]")
final_time = time.time()
log.info(
f"Execution total time = {((final_time - initial_time) / 60)} minutes")
return vectorized_issue
def main():
# Stores the execution start time to calculate the time it takes for the module to execute.
initial_time = time.time()
# Define el logger que se utilizará.
logger = log4p.GetLogger(__name__)
log = logger.logger
# Check if there is a running process that contains the name of this module.
check_same_python_module_already_running(os.path.split(__file__))
# Carga el fichero de configuración para el entorno.
env_path = Path(ROOT_DIR) / 'config' / (SYN_ENV + '.env')
load_dotenv(dotenv_path=env_path)
# Inicializa los parámetros MongoDB para almacenar las estadísticas.
mongodb_client: MongoClient = get_default_mongo_client()
db = mongodb_client["bugzilla"]
col = db["eclipse_all"]
# Campos seleccionados para los trabajos abordados en el proyecto de investigación.
fields = {
"_id": 0,
"assigned_to": 1,
"assigned_to_detail": 1,
"classification": 1,
"component": 1,
"creation_time": 1,
"creator": 1,
"creator_detail": 1,
"dupe_of": 1,
"id": 1,
"op_sys": 1,
"platform": 1,
"priority": 1,
"product": 1,
"resolution": 1,
"severity": 1,
"status": 1,
"summary": 1,
"version": 1,
"description": "$comments.text"
}
aggregation_project = {"$project": fields}
# Se utilizarán sólo las incidencias resueltas.
aggregation_match = {"$match": {
'$and': [
{
'$or': [
{'status': {'$eq': 'RESOLVED'}},
{'status': {'$eq': 'VERIFIED'}},
{'status': {'$eq': 'CLOSED'}}
]
},
{'comments.count': {'$eq': 0}}
]
}}
aggregation_unwind = {"$unwind": "$comments"}
aggregation_limit = {"$limit": 20}
# data = col.find(query, fields).limit(20)
data = col.aggregate([
aggregation_unwind,
aggregation_match,
aggregation_limit,
aggregation_project
])
# Expande el cursor y construye el DataFrame
eclipse_base = pd.DataFrame(list(data))
print(eclipse_base.head(10))
# 1º-Convertir la variable en tipo categórica:
eclipse_base.priority = eclipse_base.priority.astype('category')
print(eclipse_base.head(10))
# 2º-Catergorizar:
eclipse_base['priority_cod'] = eclipse_base['priority'].cat.codes
print(eclipse_base.head(10))
# 1- Primero creas una instancia:
# le = sklearn.preprocessing.LabelEncoder()
# 2- Después ajustas a tus datos:
# le.fit(labels) (en este caso, "labels = data[:,0]" era la columna con las predicciones).
# 3- Obtienes la columna con los valores transformados:
# labels = le.transform(labels)
# 4- Puedes guardar los valores originales para establecer una relación:
# class_names = le.classes_
# df["creation_time_year"] = df["creation_time"].str[:4]
# df["last_change_time_year"] = df["last_change_time"].str[:4]
# df["resolution_string"] = df["resolution"].apply(lambda y: "EMPTY_FIELD" if len(y) == 0 else y)
final_time = time.time()
log.info(f"Total execution time = {((final_time - initial_time) / 60)} minutos")
# -*- coding: utf-8 -*-
import json
import time
import requests
import log4p
LOG = log4p.GetLogger('DOperating').logger
API_BASE = 'https://api.tapd.cn'
class Tapd:
_api_user = None
_api_password = None
def __init__(self, api_user, api_password):
self._api_user = api_user
self._api_password = api_password
def _request_api_get(self, method, parms):
"""
网络请求GET公共类
:param method:
:param parms:
:return:
"""
time.sleep(1)
req = requests.get("{0}{1}?{2}".format(API_BASE, method, parms),
auth=(self._api_user, self._api_password))
return json.loads(req.content.decode('utf8'))
from argparse import ArgumentParser
from argparse import RawDescriptionHelpFormatter
__all__ = []
__version__ = 0.1
__date__ = '2019-09-10'
__updated__ = '2019-09-13'
__author__ = 'Klaus Tachtler <*****@*****.**>'
__organisation__ = 'Klaus Tachtler'
import Milter
import log4p
from socket import AF_INET, AF_INET6, AF_UNIX
__logger__ = log4p.GetLogger(__name__, config="./log4p.json")
__log__ = __logger__.logger
__socketName__ = None
__listener__ = "127.0.0.1"
__port__ = 10099
__timeout__ = 600
__flags__ = 0
class OutputFormater():
'''Formatter for the log output'''
def __init__(self, clipChar, lineChar, lineCharCount, listFormat,
lineFormat, errorLineFormat):
self.clipChar = clipChar
self.lineChar = lineChar
# -*- coding: utf-8 -*-
import pyodbc
import configparser
import log4p
LOG = log4p.GetLogger('Performance').logger
class Performance:
_db_conn = None
def __init__(self):
db_config = configparser.ConfigParser()
db_config.read('config.ini')
db_ip = db_config['database']['sql_ip']
db_user = db_config['database']['sql_user']
db_pass = db_config['database']['sql_password']
self._db_conn = pyodbc.connect(
'DRIVER={ODBC Driver 17 for SQL Server};SERVER=' + db_ip +
';DATABASE=DevelopProjectManaDB;UID=' + db_user + ';PWD=' +
db_pass)
def query_project(self, project_name):
"""
通过项目名获得项目编号
:param project_name:
:return:
"""
sql = 'SELECT * FROM project_dev_manage_tb WHERE project_name=?'
cursor = self._db_conn.cursor()
row = cursor.execute(sql, project_name).fetchone()
"""
import os
import importlib
import inspect
import log4p
from flask import Flask, Blueprint, request, send_from_directory
from hee.heeconfig import Config
print("heeframework start starts to initialize.")
logger_ = log4p.GetLogger(logger_name=__name__,
logging_level="INFO",
config="config/log4p.json")
log_ = logger_.logger
# Import dependencies required by built-in modules before framework initialization.
config = Config()
if config.has_section('MYSQL'):
from hee.rdb_mysql import DbMySQL
class HeeContainer:
"""
submod and object container.
"""
def __init__(self):
# all submods
def scan_and_load_submod(self, path):
"""
Scan and load all modules
This method will scan from the current directory, and if a main files is found, it will be loaded as a submodule.
After the submodule is successfully loaded, all classes under the module will be scanned and an instance of each class will be created.
The name of the instance object of this class in the container is: module name + class name.
If there is a log variable in the submodule, create a Logger object and associate it
If the submodule ends with the controller, create a route and register it in the route
Finally, all modules are loaded into the context.
"""
log_.info("scan path: %s " % path)
# Determine if it is a path, process the sub-path recursively
if os.path.isdir(path):
for subpath in os.listdir(path):
log_.info("subpath: %s" % subpath)
if subpath == 'hee':
continue
if subpath.endswith('__pycache__'):
continue
if subpath == 'static':
continue
if subpath == 'template':
continue
self.scan_and_load_submod(path + "/" + subpath)
# If it is a python file, load the processing module
elif path.endswith(".py"):
# All files under the root path are skipped
if path == 'hee_framework':
return
# Load all submods
submod_full_name = path.replace("./", "").replace("/",
".").replace(
".py", "")
log_.info('Load submod: ' + submod_full_name)
submod = importlib.import_module(submod_full_name)
# Automatic config injection
if 'config' in dir(submod):
submod.config = self.config
# Automatic log injection
if 'log' in dir(submod):
submod_logger = log4p.GetLogger(logger_name=submod_full_name,
config="config/log4p.json")
submod.log = submod_logger.logger
log_.info("submod [" + submod_full_name +
"] log has been initialized.")
# Automatic hee injection
if 'hee' in dir(submod):
submod.hee = self.hee
log_.info("submod [" + submod_full_name +
"] hee has been initialized.")
# -- flows are dynamic builtin module. --
# db_mysql dynamic module injection
if 'db' in dir(submod):
submod.db = self.dbmysql
log_.info("submod [" + submod_full_name +
"] dynamic module has been initialized.")
# save submods
self.hee_container.submods[submod_full_name] = submod
else:
log_.info("skip path: %s" % path)
#!/usr/bin/env python # -*- coding: utf-8 -*- # @Date : 2017-12-13 01:37:50 # @Author : 郝天飞/Talen Hao ([email protected]) # @Link : talenhao.github.io # @Version : $Id$ import configparser # for log >> import logging import os import log4p SCRIPT_NAME = os.path.basename(__file__) pLogger = log4p.GetLogger(SCRIPT_NAME, logging.DEBUG).get_l() # log end << config_file = os.path.dirname(os.path.abspath(__file__)) + '/config.ini' class SCEConfigParser: def __init__(self, config_file=config_file): self.config_file = config_file pLogger.debug("Use config file: {} ".format(self.config_file)) def config_parser(self): # with open(self.config_file,mode='r') as self.fp: # self.python_config_parser.read_file(self.fp) try: python_config_parser = configparser.ConfigParser() python_config_parser.read(self.config_file) except configparser.ParsingError:
# -*- coding:utf8 -*-
import getopt
import os
import sys
import time
from archive import Archive
from doperating import DOperating
from gmail import GMail
import configparser
import log4p
LOG = log4p.GetLogger('__main__').logger
def usage():
"""
打印帮助
:return:
"""
print("""usage: python3 jdoka.py [option]
-l : 使程序循环执行，默认执行间隔5分钟
-t minute : 使用此参数可以修改循环的间隔时间，只有使用-l后此参数才会生效
-h,--help : 打印此帮助
--==================--
--mail-config=path : 收发信服务器配置项目路径，如不配置为项目同目录下的conf内
--db-config=path : 数据库服务器配置项目路径，如不配置为项目同目录下的conf内
--profession-config:业务处理配置文件
--result-path=path : 结果文件路径
def main():
# Stores the execution start time to calculate the time it takes for the module to execute.
initial_time = time.time()
# Define el logger que se utilizará.
logger = log4p.GetLogger(__name__)
log = logger.logger
# Check if there is a running process that contains the name of this module.
check_same_python_module_already_running(os.path.split(__file__))
# Carga el fichero de configuración para el entorno.
env_path = Path(ROOT_DIR) / 'config' / (SYN_ENV + '.env')
load_dotenv(dotenv_path=env_path)
# Inicializa los parámetros MongoDB para almacenar las estadísticas.
mongodb_client: MongoClient = get_default_mongo_client()
db = mongodb_client["bugzilla"]
col = db["eclipse_all"]
# Campos seleccionados para los trabajos abordados en el proyecto de investigación.
fields = {
"_id": 0,
"assigned_to": 1,
"assigned_to_detail": 1,
"classification": 1,
"component": 1,
"creation_time": 1,
"creator": 1,
"creator_detail": 1,
"dupe_of": 1,
"id": 1,
"op_sys": 1,
"platform": 1,
"priority": 1,
"product": 1,
"resolution": 1,
"severity": 1,
"status": 1,
"summary": 1,
"version": 1,
"description": "$comments.text"
}
aggregation_project = {"$project": fields}
# Se utilizarán sólo las incidencias resueltas.
aggregation_match = {"$match": {
'$and': [
{
'$or': [
{'status': {'$eq': 'RESOLVED'}},
{'status': {'$eq': 'VERIFIED'}},
{'status': {'$eq': 'CLOSED'}}
]
},
{'comments': {'$exists': 'true'}},
{'comments': {'$ne': 'null'}},
{'comments': {'$ne': ""}},
{'comments': {'$not': {'$size': 0}}},
{'comments.count': {'$eq': 0}}
]
}}
aggregation_unwind = {"$unwind": "$comments"}
aggregation_limit = {"$limit": 20}
# data = col.find(query, fields).limit(20)
data = col.aggregate([
aggregation_unwind,
aggregation_match,
# aggregation_limit,
aggregation_project
])
# Expande el cursor y construye el DataFrame
eclipse_base = pd.DataFrame(list(data))
# Almacena el dataframe en MongoDB.
db["eclipse_base"].insert_many(eclipse_base.to_dict('records'))
final_time = time.time()
log.info(f"Total execution time = {((final_time - initial_time) / 60)} minutos")
def main():
# Stores the execution start time to calculate the time it takes for the module to execute.
initial_time = time.time()
# Define el logger que se utilizará.
logger = log4p.GetLogger(__name__)
log = logger.logger
# Check if there is a running process that contains the name of this module.
check_same_python_module_already_running(os.path.split(__file__))
# Carga el fichero de configuración para el entorno.
env_path = Path(ROOT_DIR) / 'config' / (SYN_ENV + '.env')
load_dotenv(dotenv_path=env_path)
# Inicializa los parámetros MongoDB para almacenar las estadísticas.
mongodb_client: MongoClient = get_default_mongo_client()
db = mongodb_client["eclipse"]
col = db["clear"]
# Campos seleccionados para los trabajos abordados en el proyecto de investigación.
fields = {
"_id": 0,
"bug_id": 1,
# "product": 1,
# "description": 1,
# "bug_severity": 1,
# "dup_id": 1,
"short_desc": 1,
# "priority": 1,
# "version": 1,
# "component": 1,
# "delta_ts": 1,
"bug_status": 1
# "creation_ts": 1,
# "resolution": 1
}
# Se utilizarán sólo las incidencias resueltas.
query = {
'$or': [{
'bug_status': {
'$eq': 'RESOLVED'
}
}, {
'bug_status': {
'$eq': 'VERIFIED'
}
}]
}
data = col.find(query, fields)
# .limit(20)
# Expande el cursor y construye el DataFrame
clear_nlp = pd.DataFrame(list(data))
print(clear_nlp)
nltk.download("stopwords", quiet=True)
clear_nlp["short_desc_split_alpha"] = clear_nlp["short_desc"].apply(
lambda x: clean_doc_split(x))
clear_nlp["short_desc_lower"] = clear_nlp["short_desc_split_alpha"].apply(
lambda x: clean_doc_lower(x))
clear_nlp["short_desc_punctuaction"] = clear_nlp["short_desc_lower"].apply(
lambda x: clean_doc_punctuaction(x))
clear_nlp["short_desc_trim"] = clear_nlp["short_desc_punctuaction"].apply(
lambda x: clean_doc_trim(x))
clear_nlp["short_desc_isalpha"] = clear_nlp["short_desc_trim"].apply(
lambda x: clean_doc_isalpha(x))
clear_nlp["short_desc_stop_words"] = clear_nlp["short_desc_isalpha"].apply(
lambda x: clean_doc_stopW(x))
clear_nlp["short_desc_diacritic"] = clear_nlp[
"short_desc_stop_words"].apply(lambda x: clean_doc_diacri(x))
clear_nlp["short_desc_lemmatizer"] = clear_nlp[
"short_desc_diacritic"].apply(lambda x: clean_doc_lem(x))
# Elimina la columna 'bug_status' porque se realizará después un merge con la colección original.
clear_nlp.drop('bug_status', axis=1, inplace=True)
# Almacena el dataframe en MongoDB.
db["clear_short_desc_nlp"].insert_many(clear_nlp.to_dict('records'))
final_time = time.time()
log.info(
f"Total execution time = {((final_time - initial_time) / 60)} minutos")
import dynet as dy
import log4p
from dotenv import load_dotenv
from definitions import ROOT_DIR, SYN_ENV
from syn.model.build.codebooks.incidences import data
from syn.model.build.codebooks.incidences.tasks import Task
from syn.model.build.treelstm.tree import TreeLstm, TreeLstmCategorical
from syn.model.build.treelstm.vectorizer.Vectorizer import get_vectorized_issue
env_path = Path(ROOT_DIR) / 'config' / (SYN_ENV + '.env')
load_dotenv(dotenv_path=env_path)
# Define el logger que se utilizará.
logger = log4p.GetLogger(__name__)
log = logger.logger
class TreeLstmDuplicateTrain(Task):
"""The task of training a duplicate detector"""
def __init__(self,
corpus='bugzilla',
collection='clear',
architecture='LSTM',
attention=True,
attention_size=10,
glove_size=100,
hidden_size=100,
max_input=200,
batch_size=1,
