def _get_removed_organizations_and_filials_by_ids(ids, progress, verbose=False):
if progress == 'ipython':
from utility.ipython_utility import log_progress
organizations = []
filials = []
session = requests.Session()
for i in log_progress(ids, name='Идентификатор'):
o, f = _get_removed_organization_and_filial_by_ids(i, session, verbose)
if o is not None:
organizations.append(o)
if f is not None:
filials.append(f)
return np.unique(filials), np.unique(organizations)
else:
from tqdm import tqdm
organizations = []
filials = []
session = requests.Session()
for i in tqdm(ids):
o, f = _get_removed_organization_and_filial_by_ids(i, session, verbose)
if o is not None:
organizations.append(o)
if f is not None:
filials.append(f)
return np.unique(filials), np.unique(organizations)
def get_organizations_and_filials_by_buildings(buildings, progress, verbose=False):
if progress == 'ipython':
from utility.ipython_utility import log_progress
organizations = []
filials = []
session = requests.Session()
for building in log_progress(buildings, name='Здания'):
orgs, fils = _get_organizations_and_filials_by_buildings(building, session, verbose)
organizations += orgs
filials += fils
return np.unique(organizations), np.unique(filials)
else:
from tqdm import tqdm
organizations = []
filials = []
session = requests.Session()
for building in tqdm(buildings):
orgs, fils = _get_organizations_and_filials_by_buildings(building, session, verbose)
organizations += orgs
filials += fils
return np.unique(organizations), np.unique(filials)
def filter_by_categories(filials, organizations,
main_categories_filter_ids, sub_categories_filter_ids, progress='ipython'):
if progress == 'ipython':
from utility.ipython_utility import log_progress
filtered = []
for filial in log_progress(filials, name='Филиалы', every=1):
if _filial_valid(filial, organizations, main_categories_filter_ids, sub_categories_filter_ids):
filtered.append(filial)
return np.array(filtered)
else:
from tqdm import tqdm
filtered = []
for filial in tqdm(filials):
if _filial_valid(filial, organizations, main_categories_filter_ids, sub_categories_filter_ids):
filtered.append(filial)
return np.array(filtered)
def get_buildings_by_radius(clusters, radius_meters=250, verbose=False, progress='ipython'):
if progress == 'ipython':
from utility.ipython_utility import log_progress
buildings = []
session = requests.Session()
for cluster in log_progress(clusters, name='Кластеры'):
b = _get_building(cluster, session, radius_meters, verbose)
if b is not None:
buildings.append(b)
return buildings
else:
from tqdm import tqdm
buildings = []
session = requests.Session()
for cluster in tqdm(clusters):
b = _get_building(cluster, session, radius_meters, verbose)
if b is not None:
buildings.append(b)
return buildings
def update_filials_dates(filials, progress='ipython', verbose=False):
if progress == 'ipython':
from utility.ipython_utility import log_progress
updated_filials = []
session = requests.Session()
for filial in log_progress(filials, name='Организации'):
updated_filial = _update_filial_date(filial, session, verbose)
updated_filials.append(updated_filial)
return np.unique(updated_filials)
else:
from tqdm import tqdm
updated_filials = []
session = requests.Session()
for filial in tqdm(filials):
updated_filial = _update_filial_date(filial, session, verbose)
updated_filials.append(updated_filial)
return np.unique(updated_filials)
def get_filials_by_organizations(organizations, verbose=False, progress='ipython'):
if progress == 'ipython':
from utility.ipython_utility import log_progress
filials = []
session = requests.Session()
for organization in log_progress(organizations, name='Организации'):
fils = _get_filials_by_organization(organization, session, verbose)
filials += fils
return np.unique(filials)
else:
from tqdm import tqdm
filials = []
session = requests.Session()
for organization in tqdm(organizations):
fils = _get_filials_by_organization(organization, session, verbose)
filials += fils
return np.unique(filials)
def filter_by_stop_words(filials,
organizations,
stop_words,
progress='ipython'):
if progress == 'ipython':
from utility.ipython_utility import log_progress
filtered = []
for filial in log_progress(filials, name='Филиалы', every=1):
org = next(o for o in organizations
if o.id == filial.organization_id)
if not _filial_name_in_stop(org.name, stop_words):
filtered.append(filial)
return np.array(filtered)
else:
from tqdm import tqdm
filtered = []
for filial in tqdm(filials):
if _filial_name_in_stop(filial.name, stop_words):
filtered.append(filial)
return np.array(filtered)
def export_excel(api, progress='ipython'):
if progress == 'ipython':
from utility.ipython_utility import log_progress
data = []
for filial in log_progress(api.filials, name='Филиалы'):
row = _process_element(api, filial)
if row is not None:
data.append(row)
df = pd.DataFrame(data=data,
columns=['category', 'company', 'address', 'latitude', 'longtitude', 'nearest_name',
'nearest_metro_lat', 'nearest_metro_long', 'nearest_distance',
'created_date', 'removed_date', 'living_period', 'current_status']) \
.rename(columns={'category': 'Отрасль',
'company': 'Компания',
'address': 'Адрес',
'latitude': 'Широта_к',
'longtitude': 'Долгота_к',
'nearest_name': 'Ближайшее метро',
'nearest_metro_lat': 'Широта_м',
'nearest_metro_long': 'Долгота_м',
'nearest_distance': 'Расстояние до метро',
'created_date': 'Дата открытия',
'removed_date': 'Дата закрытия',
'living_period': 'Срок жизни, мес.',
'current_status': 'Статус'})
if not os.path.exists('data/files/excel_exported/'):
os.makedirs('data/files/excel_exported/')
df.to_excel('data/files/excel_exported/exported_{0}.xlsx'.format(datetime.now().strftime('%Y_%m_%d-%H_%M_%S ')))
return df
else:
from tqdm import tqdm
data = []
for filial in tqdm(api.filials):
row = _process_element(api, filial)
if row is not None:
data.append(row)
df = pd.DataFrame(data=data,
columns=['category', 'company', 'address', 'latitude', 'longtitude', 'nearest_name',
'nearest_metro_lat', 'nearest_metro_long', 'nearest_distance',
'created_date', 'removed_date', 'living_period', 'current_status']) \
.rename(columns={'category': 'Отрасль',
'company': 'Компания',
'address': 'Адрес',
'latitude': 'Широта_к',
'longtitude': 'Долгота_к',
'nearest_name': 'Ближайшее метро',
'nearest_metro_lat': 'Широта_м',
'nearest_metro_long': 'Долгота_м',
'nearest_distance': 'Расстояние до метро',
'created_date': 'Дата открытия',
'removed_date': 'Дата закрытия',
'living_period': 'Срок жизни, мес.',
'current_status': 'Статус'})
if not os.path.exists('data/files/excel_exported/'):
os.makedirs('data/files/excel_exported/')
df.to_excel('data/files/excel_exported/exported_{0}.xlsx'.format(datetime.now().strftime('%Y_%m_%d-%H_%M_%S ')))
return df