def _create_task(self): if not self.results: time_lapsed = round(time.time() - self.time_start) if time_lapsed > self.results_wait: self.terminate_counter += 1 logger.debug("Waiting to terminate {}/{}".format( self.terminate_counter, self.termination_limit)) return self._tasks_lock.acquire() task_results = self.results[:self.results_len] self.results = self.results[self.results_len:] try: result_ids, versions = zip(*task_results) result_ids = sorted(set(result_ids)) model_version = min(versions) self.curr_task_id += 1 task_name = f"task-{self.curr_task_id}-model-{model_version}" result_ids = [p for p in result_ids if os.path.exists(p) and (os.path.basename(p) not in self.seen_results)] [self.seen_results.add(os.path.basename(p)) for p in result_ids] if not result_ids: self.terminate_counter += 1 return with self.train_lock: task_id = self.cli.tasks_create(task_name, self.labels, result_ids) self.pending_tasks[task_id] = result_ids self.time_start = time.time() except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError, requests.exceptions.RequestException) as e: self._tasks_lock.release() logger.critical(e)
def authenticate(email, password): auth_url = 'https://api.jinka.fr/apiv2/user/auth' auth_dict = {'email':email, 'password':password} s = requests.Session() r_auth = s.post(auth_url, auth_dict) if r_auth.status_code == 200: logger.info('Authentification succeeded (200)') access_token = r_auth.json()['access_token'] else: logger.critical(f'Authentification failed with error {r_auth.status_code}') return None, None headers = { 'Accept': '*/*', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36', 'Accept-Language': 'fr,fr-FR;q=0.8,en-US;q=0.5,en;q=0.3', 'Content-Type': 'application/json', 'Authorization': f'Bearer {access_token}', 'Origin': 'https://www.jinka.fr', 'Connection': 'keep-alive', 'DNT': '1', 'Sec-GPC': '1', 'If-None-Match': 'W/f46-qWZd5Nq9sjWAv9cj3oEhFaxFuek', 'TE': 'Trailers', } return s, headers
def translate( self, source: str, from_lang: Optional[str] = None, to_lang: Optional[str] = None, proxy: Optional[str] = None, ) -> str: ''' if from_lang to_lang: None, use instance's if proxy != self.proxy: use new client ''' if from_lang is None: self.from_lang_ = self.from_lang else: self.from_lang_ = from_lang if to_lang is None: self.to_lang_ = self.to_lang else: self.to_lang_ = to_lang if self.to_lang_ == self.from_lang_: return source # self.source_list = wrap(source, 1000, replace_whitespace=False) # last "." not converted to "。" in dest chinese, attache ' _xx', remove 'XXX' [:-4] if to_lang == 'zh': # source_list = wrap(source + ' . _xx', 500, replace_whitespace=False) # a chinese char about 3 bytes source_list = wrap(source + ' . _xx', 160, replace_whitespace=False) else: source_list = wrap(source, 500, replace_whitespace=False) # self.source_list = wrap(source, 1695, replace_whitespace=False) # MAX ALLOWED QUERY : 500 bytes/CHARS try: _ = (self._get_translation(elm, proxy=proxy) for elm in source_list) seq = ' '.join(_) except Exception as exc: logger.warning(" seq = ' '.join, exc %s ", exc) # return None raise # if seq[:17] == 'MYMEMORY WARNING:': if seq.startswith(('MYMEMORY WARNING:', )): logger.critical("MYMEMORY WARNING: %s", seq) raise QuotaError(seq) # last "." not converted to "。" in dest chinese, attache ' _xx', remove 'XXX' [:-4] if self.to_lang == 'zh': return seq[:-5] return seq
def connect(self): self.connect = False try: print('~'*100 + f'\nConnecting to device: {self.ip}') self.connect = ConnectHandler(ip=self.ip, device_type='cisco_ios', username=self.username, password=self.password) except (NetMikoAuthenticationException, NetMikoTimeoutException) as conn_error: logger.warning(f'Unable to connect to device:\n{conn_error}') if not self.connect: logger.critical('No devices to connect to') sys.exit()
def foo(): logfile("/home/kaktus74/nasze_logi.log") for i in range(1, 100): logger.debug("SUper szczegolowe informacje") logger.info("Cos siw wydarzylo!") logger.warning("Chyba cos sie wywrocilo") logger.error("Oj, cos sie na pewno wywrocilo!") rok = 19 logger.critical(f"Wykryto wirusa covid-{rok}! {i}")
def include_all_items_in_live( self ): # If some items are "excluded" from the live plan, it will break the service planner app's item order. Run this to make all items "included" in the live plan. r = requests.get( f'https://api.planningcenteronline.com/services/v2/service_types/' f'{self.service_type}/plans/{self.plan_id}/items/?&include=item_times', auth=(APP_ID, SECRET)) r = r.json() excluded = [ ] # /service_types/#/plans/#/items/#/item_times/id // list of dicts of item time ids that are excluded for item_time in r['included']: if item_time['attributes']['exclude'] is True: item = { 'item_id': item_time['relationships']['item']['data']['id'], 'time_id': item_time['id'] } excluded.append(item) if len(excluded) > 0: request_headers = { 'Content-type': 'application/json', 'Accept': 'text/plain' } payload = { "data": { "type": "ItemTime", "attributes": { "exclude": False, }, } } for item in excluded: r = requests.patch( f'https://api.planningcenteronline.com/services/v2/service_types/' f'{self.service_type}/plans/{self.plan_id}/items/{item["item_id"]}/item_times/{item["time_id"]}', headers=request_headers, data=json.dumps(payload), auth=(APP_ID, SECRET)) if r.status_code != 200: logger.critical( 'pco_plan.include_all_items_in_live: ERROR including item time %s', item) else: logger.info( 'pco_plan.include_all_items_in_live: updated live item to be included in plan: %s', item)
def devices_connect(devices): connect = False n = int(0) for device in devices: try: print('~'*100 + f'\nConnecting to device: {device}') connect[n] = ConnectHandler(ip=device, device_type='cisco_ios', username=username, password=password) n = n+1 except (NetMikoAuthenticationException, NetMikoTimeoutException) as conn_error: logger.warning(f'Unable to connect to device:\n{conn_error}') if not connect: logger.critical('No devices to connect to') sys.exit()
def remove_expired(session, df, last_deleted_path): df_expired = df.loc[df["expired_at"].notna(), :] if len(df_expired)>15: logger.critical('Df slicing error') exit() logger.info('Starting the cleaning of expired offers.') for appart_id, row in tqdm(df_expired.iterrows()): post_url = 'https://api.jinka.fr/apiv2/alert/' + row['alert_id'] + '/abuses' data = {'ad_id':appart_id, 'reason':'ad_link_404'} session.post(post_url, data=data) df_expired.to_json(last_deleted_path, orient='columns') cleaned_df = df.loc[df['expired_at'].isna(), :] logger.info(f'Finished cleaning the {len(df_expired)} expired appartments.') return cleaned_df
def download_template(self): logger.debug('trying to connect to GITLAB') try: git_req = requests.get('http://172.17.0.3/api/v4/projects/1/repository/files/router/raw?ref=master', headers={'PRIVATE-TOKEN': 'x6sP6xf57gb5sxxiXutq'}) except requests.exceptions.RequestException as req_error: logger.critical(f'unable to connect to GITLAB:\n{req_error}') return False if git_req.status_code == 200: logger.debug('connection to GITLAB successful') self.template = git_req.text.splitlines() return True else: logger.info('Problem with downloading the template') return False
def readConfig(): configFile = "config.yml" try: with open(configFile, "r") as configString: parsedConfig = load(configString, Loader=FullLoader) log.debug("Read config file {0}, got {1}".format( configFile, parsedConfig)) return parsedConfig except FileNotFoundError: log.critical("Can't read {0}!".format(configFile)) print("Konnte die Configdatei nicht laden!") log.info("Program end.") sys.exit(1)
async def connect_retry(self): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setblocking(False) while True: try: await self.loop.sock_connect(sock, self.address) logger.info( f"Connected to {self.address[0]}:{self.address[1]}.") return sock except (ConnectionRefusedError, ConnectionAbortedError, socket.gaierror, OSError): logger.critical( f"Connect call to {self.address[0]}:{self.address[1]} failed, retrying in {self.retry_timeout} " "second(s).") await asyncio.sleep(self.retry_timeout)
async def connect(self): while True: try: self.conn = await r.connect(self.addr, self.port) sock = self.conn._instance._streamwriter.get_extra_info( 'socket') print(sock) set_keepalive(sock) break except r.ReqlDriverError: logger.critical( f"Failed to connect to database, retrying in {self.retry_timeout} seconds." ) await asyncio.sleep(self.retry_timeout) self.connection_available.set() self.connection_available.clear()
def receive(message): try: type = message[Keys.TYPE] if type not in Types.all: logger.critical(f"Unknown message type {type}") return if type not in dispatch_table: logger.debug( f"Message type {type} does not have an handler.") return handler = dispatch_table[type] asyncio.ensure_future(handler(self, stream, message)) except: logger.exeption("Unexpected exception in message handling")
def __prepare(self, infile, cities): scripture = MongoClient(settings.MONGO).scripture df = pandas.read_csv(infile, error_bad_lines=False) # keys = df.keys().tolist() cities = cities and Cities.from_json(cities) or Cities() cities.hub = MongoClient(settings.HUB_MONGO).hub for idx, row in df.iterrows(): isnull = row.isnull() if isnull.get('ta'): continue bonotel = not isnull.get('bonotel', True) and \ int(row.bonotel) or None roomsxml = not isnull.get('roomsxml', True) and \ int(row.roomsxml) or None hotelspro = not isnull.get('hotelspro', True) and \ str(row.hotelspro) or None hotelbeds = not isnull.get('hotelbeds', True) and \ int(row.hotelbeds) or None jactravel = not isnull.get('jactravel', True) and \ int(row.jactravel) or None ta = row.get('ta') hcom = row.get('hotel_URL') if pandas.isnull(hcom): continue try: hcom_id = self.__parse_hcom_id(hcom) except AttributeError: continue try: _city = row.get('city') or row.get('城市') city_name = self._CITY_EN_RE.match(_city).group(1) city = cities.name(city_name.strip()) yield Hotel( db=scripture, hotel_id=roomsxml, hotels_cn_id=hcom_id, bonotel=bonotel, hotelspro=hotelspro, hotelbeds=hotelbeds, jactravel=jactravel, comments_url=ta, city=city ) except (KeyError, IndexError, TypeError) as e: logger.critical('hcom(%s), city(%s)', hcom_id, _city) logger.exception(e)
def _check_status(self): with self.train_lock: task_ids = self.pending_tasks.keys() if not task_ids: return try: for i in self._get_completed_tasks(task_ids): task_id, image_ids = self.cli.tasks_dump(i) self._add_train(task_id, image_ids) # updating task start time to prevent a new task # creation immediatelly after releasing lock self.time_start = time.time() self._tasks_lock.release() except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError, requests.exceptions.RequestException) as e: logger.critical(e)
def interrupt_experiment_on_unhealthy_probe(probe: Probe, run: Run, configuration: Configuration, secrets=Secrets) -> None: if experiment_finished.is_set(): return tolerance = probe.get("tolerance") checked = within_tolerance(tolerance, run["output"], configuration=configuration, secrets=secrets) if not checked and not guardian.interrupted: guardian.interrupted = True if not experiment_finished.is_set(): logger.critical( "Safeguard '{}' triggered the end of the experiment".format( probe["name"])) exit_gracefully()
def run_all(email, password, expired): s, headers = authenticate(email, password) if s == None: logger.critical('Aborting search, check your credentials.') quit() df_alerts = get_alerts(s, headers) df_apparts, expired_index = get_all_apparts(df_alerts, s, headers) df_apparts = cleaner(df_apparts) df_apparts = features_engineering(df_apparts) df_history = append_history_df(df_apparts, HISTORY_PATH) df_apparts = df_apparts.loc[~df_apparts.index.duplicated()] df_apparts = get_all_links(s, df_apparts, expired, APPARTS_DB_PATH) if expired: df_history = update_history_df(df_apparts, df_history, expired_index) df_apparts = remove_expired(s, df_apparts, LAST_DELETED_PATH) df_apparts.to_csv(APPARTS_CSV_PATH, sep=';', encoding='utf-8') with warnings.catch_warnings(): warnings.simplefilter("ignore") try: df_apparts.to_excel(APPARTS_XLSX_PATH, encoding='utf-8') except IllegalCharacterError as e: logger.warn( "Some illegal characters were replaced in the dataframe.") ILLEGAL_CHARACTERS_RE = re.compile( r'[\000-\010]|[\013-\014]|[\016-\037]') df_apparts.applymap(lambda x: ILLEGAL_CHARACTERS_RE.sub(r'', x) if isinstance(x, str) else x).to_excel( APPARTS_XLSX_PATH, encoding='utf-8') df_history.to_csv(HISTORY_PATH, sep=';', encoding='utf-8') if upload: uploader = Uploader(credentials_path=CREDS_PATH, token_file_path=TOKEN_FILE_PATH, secret_client_path=SECRET_CLIENT_PATH) uploader.push_table( df_apparts, spreadsheet_id='131UoWqQwZfydMJ3yqVe-L6TY6NKtJx8zVNppo034dT4', worksheet_name='apparts', index=True)
def process_failure(self, failure, message_id, sign_method, sign_params, request_counter): if not isinstance(failure.value, ServiceException): # All handled exceptions should inherit from ServiceException class # Throwing other exception class means that it is unhandled error # and we should log it logger.critical(failure) sign = False code = getattr(failure.value, "code", -1) if message_id is not None: # Other party does not care of error state for notifications if os.environ.get("debug"): tb = failure.getBriefTraceback() else: tb = None self.writeJsonError(code, failure.getErrorMessage(), tb, message_id, sign, sign_method, sign_params) request_counter.decrease()
def push_standard_config(): logger.debug('trying to connect to GITLAB') try: git_req = requests.get('http://172.17.0.3/api/v4/projects/1/repository/files/start/raw?ref=master', headers={'PRIVATE-TOKEN': 'x6sP6xf57gb5sxxiXutq'}) except requests.exceptions.RequestException as req_error: logger.critical(f'unable to connect to GITLAB:\n{req_error}') return False logger.debug('checking if connection was successful') if git_req.status_code == 200: logger.debug('connection to GITLAB successful') out = git_req.text.splitlines() if isinstance(out, list): logger.debug(f'about to send this configuration to device:\n{out}') logger.debug(f'connecting to devices:\n{devices_ip}') devices_connect(devices=devices_ip_strip) try: abc = devices_connect. sendconf = connect.send_config_set(out) except Exception as sendconf_error: logger.critical(sendconf_error) return False if (re.search('\^', sendconf)) is not None: conf_apply_error = ConfigInputException('Error in some command! Check log') logger.critical(f'{conf_apply_error} output from device:\n{sendconf}') raise conf_apply_error else: conf_success = 'Configuration applied successfully' logger.info(conf_success + f'output from device:\n{sendconf}') return True else: list_error = AttributeError('Problem with converting template to list') logger.critical(list_error) raise list_error else: logger.info('Problem with downloading the template') return False
def push_template(self): if isinstance(self.template, list): logger.debug(f'about to send this configuration to device:\n{self.template}') logger.debug(f'connecting to device:\n{self.ip}') try: sendconf = self.connect.send_config_set(self.template) except Exception as sendconf_error: logger.critical(sendconf_error) return False if (re.search('\^', sendconf)) is not None: conf_apply_error = ConfigInputException('Error in some command! Check log') logger.critical(f'{conf_apply_error} output from device:\n{sendconf}') raise conf_apply_error else: conf_success = 'Configuration applied successfully' logger.info(conf_success + f'output from device:\n{sendconf}') return True else: list_error = AttributeError('Problem with converting template to list') logger.critical(list_error) raise list_error
async def print_loop() -> None: """main loop - checks if entries exist periodically and prints them""" await client.wait_until_ready() # setup global variables guilds = list(iter(client.guilds)) if len(guilds) != 1: logger.critical("This bot should only be used on one server") await client.logout() sys.exit(1) channels = guilds[0].channels client.feed_channel = get(channels, name="feed") client.nsfw_feed_channel = get(channels, name="nsfw-feed") if client.feed_channel is None: logger.critical("Couldn't find the 'feed' channel") if client.nsfw_feed_channel is None: logger.critical("Couldn't find the 'nsfw-feed' channel") client.old_db = OldDatabase(filepath=old_db_file) client.loop.create_task(export_loop()) while not client.is_closed(): # if there are new entries, print them await print_new_embeds() logger.debug(f"Sleeping for {client.period}") await sleep(client.period)
from pathlib import Path from collections import defaultdict, namedtuple from logzero import logger from PyTexturePacker import Packer from PyTexturePacker.MaxRectsPacker.MaxRectsPacker import MaxRectsPacker from pymongo import MongoClient from settings import settings try: from sh import crunch, ErrorReturnCode except: crunch = lambda *args, **kwargs: logger.critical( "Please install crunch for PNG optimization" ) try: from sh import cwebp except: cwebp = lambda *args, **kwargs: logger.critical( "Please install cwebp for WEBP compression" ) RE_DIM = re.compile(r"\d+") Rect = namedtuple("Rect", ["x", "y", "w", "h"]) Dimensions = namedtuple("Dimensions", ["w", "h"]) Point = namedtuple("Point", ["x", "y"])
from logzero import logger logger.critical('critical') logger.error('error') logger.warning('warning') logger.info('info') logger.debug('debug')
logzero.formatter(logzero.LogFormatter(datefmt="%Y-%m-%d %H:%M:%S")) if args["--log"]: logzero.logfile( args["--log"], encoding="utf-8", formatter=logzero.LogFormatter(datefmt="%Y-%m-%d %H:%M:%S", color=False), ) if args["--format"] == "grobid": parse_file = parse_tei_file elif args["--format"] == "parscit": parse_file = parse_parscit else: log.critical(f"Unknown --format: {args['--format']}") exit(1) cited_years = {} for dirname in args["<dir>"]: if not os.path.exists(dirname): log.error(f"Directory not found: {dirname}") continue dir_diff, dir_files, total_files = 0, 0, 0 for filename in glob(f"{dirname}/*.xml"): base = os.path.basename(filename) file_id = base.split(".")[0] if file_id.endswith("-parscit"): file_id = file_id[:-8] log.debug(f"Parsing {base}") cited_years[file_id], diff = parse_file(filename)
logzero.loglevel(loglevelFromCli) # Do we want to log as json? if (jsonLogFromCli == "Y" or jsonLogFromCli == "YES"): logzero.json() log.debug("Command Line Parameters: {0}".format(args)) # Load config configFile = "config.yml" try: with open(configFile, "r") as configString: config = yaml.load(configString, Loader=yaml.FullLoader) except FileNotFoundError: log.critical("Can't read {0}!".format(configFile)) exit(1) log.info("Library Name: {0}".format(config["libraryName"])) log.info("Locales to generate: {0}".format(config["languages"])) log.info("Default locale: {0}".format(config["defaultLanguage"])) log.info("Timezone: {0}".format(config["timezone"])) # Defaults sourceFile = args["source"] if args[ "source"] else "~/library-media-inventory/inventory.csv" log.info("Source file: {0}".format(sourceFile)) ## Some variables, init jinja2 # Current folder workDir = os.path.dirname(os.path.realpath(__file__))
def main(args): """ Let's a go... """ # set log level if args.verbose > 0: logzero.loglevel(logging.DEBUG) else: logzero.loglevel(logging.INFO) mode = "" # are we dealing with a file or a directory? if os.path.isdir(args.path): mode = "DIR" if not args.path.endswith('/'): args.path += '/' elif os.path.isfile(args.path): mode = "FILE" else: logger.critical("Invalid file or directory path specified.") # check and set output dir if necessary otherwise # we use the input dir to write our new files to # if the output_dir is specified we create dirs as required # # if it isn't and we are dealing with a directory of files # then we just make sure the input has a trailing slash # # if we are in file mode we'll and we have no output_dir # specified we'll use the input path to the file output_dir = "" if args.output_dir: output_dir = args.output_dir if not output_dir.endswith('/'): output_dir += '/' # create any dirs we need to if not os.path.exists(os.path.dirname(output_dir)): try: os.makedirs(os.path.dirname(output_dir)) except OSError as exc: # Guard against race condition if exc.errno != errno.EEXIST: raise elif mode == "DIR": output_dir = args.path if not output_dir.endswith('/'): output_dir += '/' # directory so let's loop over and do 'em all if mode == "DIR": logger.debug("Directory mode, scanning: " + args.path) # now let's get to the files and extract the transcripts from each directory = os.fsencode(args.path) for file in os.listdir(directory): filename = os.fsdecode(file) full_path = output_dir + filename if filename.endswith(".vtt"): logging.debug("Attempting to extract from file: " + filename) plaintext = extract_plaintext_from_webvtt(args.path + filename) output_string_to_file(plaintext, full_path, 'txt') continue else: logger.debug("Skipping file: " + full_path) continue elif mode == "FILE": # single file specified so just the one extraction to do logger.debug("Single file mode, scanning: " + args.path) if args.output_dir: # need to split the filename from the path as output_dir has been specified path, filename = os.path.split(args.path) output_path = output_dir + filename else: # we're plopping the files from whence they came so just use the input path to the file output_path = args.path try: with open(args.path): plaintext = extract_plaintext_from_webvtt(args.path) output_string_to_file(plaintext, output_path, 'txt') except IOError: logger.error("Input file not accessible, please check the path: " + args.filepath)
def _initialize_bucket_structure(self): """ This function creates the base structure for the bucket regarding this experiment. """ storage_creator = StorageCreator(self.storage_config) storage_object = storage_creator.build_storage_object() # Dump some test data FILE_TEST_SIZE = int(5e6) # Approx 5MB MAX_RETRIES = 2 retries = 0 file_path = os.path.join(self.experiment_dir, "initialization-service-test") generate_big_random_bin_file(file_path, FILE_TEST_SIZE) while retries < 2: # Perform Storage Connection Test logger.info("####### Testing Connection to Storage Endpoint ....") storage_object.create_bucket(DEFAULT_BUCKET_NAME) try: storage_interfacer = JobStorageInterface( storage_obj=storage_object) storage_interfacer.put_job_data( bucket=DEFAULT_BUCKET_NAME, username=self.username, project_id=self.project_name, experiment_id=self.experiment_id, variant="Initialization Service", job_id="inital_testing", local_path=file_path) logger.info("File Upload to Storage Endpoint Succeeded....") break except ConnectionError as e: logger.error( "File Upload failed, please check to storage config ....\m" f"{e}") if retries == MAX_RETRIES - 1: import sys logger.critical("\nProblem connecting to Storage. Exiting") sys.exit(status="Problem connecting to Storage. Exiting") else: logger.error( f"Trying to reconnect. {MAX_RETRIES - retries} remaining" ) retries += 1 continue file_path = os.path.join(self.experiment_dir, "initialization-service-test_restore") try: storage_interfacer.get_job_data(bucket=DEFAULT_BUCKET_NAME, username=self.username, project_id=self.project_name, experiment_id=self.experiment_id, variant="Initialization Service", job_id="inital_testing", local_path=file_path) logger.info("File Download from Storage Endpoint Succeeded....") except ConnectionError as e: logger.warning( "File Download failed, connection might be unstable ....\m" f"{e}")
def one(self, crawled_hotel, collection_name): if "en" not in crawled_hotel: if "country" not in crawled_hotel: logger.error("Country is missing. ObjectId(%s)", crawled_hotel["_id"]) return None country_name = crawled_hotel["country"] if "city" not in crawled_hotel: logger.error("City is missing. ObjectId(%s)", crawled_hotel["_id"]) return None city = crawled_hotel["city"] if "name" not in crawled_hotel: logger.error("Name is missing. ObjectId(%s)", crawled_hotel["_id"]) return None name = crawled_hotel["name"] if "address" not in crawled_hotel: logger.error("Address is missing. ObjectId(%s)", crawled_hotel["_id"]) return None address = crawled_hotel["address"] if "latitude" not in crawled_hotel: logger.error("Latitude is missing. ObjectId(%s)", crawled_hotel["_id"]) return None latitude = crawled_hotel["latitude"] if "longitude" not in crawled_hotel: logger.error("longitude is missing. ObjectId(%s)", crawled_hotel["_id"]) return None longitude = crawled_hotel["longitude"] country = self._find_country_by_cn_name(country_name) else: if "country" not in crawled_hotel["en"]: logger.error("Country is missing. ObjectId(%s)", crawled_hotel["_id"]) return None country_name = crawled_hotel["en"]["country"] if "city" not in crawled_hotel["en"]: logger.error("City is missing. ObjectId(%s)", crawled_hotel["_id"]) return None city = crawled_hotel["en"]["city"] if "name" not in crawled_hotel["en"]: logger.error("Name is missing. ObjectId(%s)", crawled_hotel["_id"]) return None name = crawled_hotel["en"]["name"] if "address" not in crawled_hotel["en"]: logger.error("Address is missing. ObjectId(%s)", crawled_hotel["_id"]) return None address = crawled_hotel["en"]["address"] if "latitude" not in crawled_hotel["en"]: logger.error("Latitude is missing. ObjectId(%s)", crawled_hotel["_id"]) return None latitude = crawled_hotel["en"]["latitude"] if "longitude" not in crawled_hotel["en"]: logger.error("Longitude is missing. ObjectId(%s)", crawled_hotel["_id"]) return None longitude = crawled_hotel["en"]["longitude"] country = self._find_country_by_en_name(country_name) if not country: country = self._find_country_by_partial_name(country_name) if not country: logger.error('Country("%s") of Hotel(%s) not found.', country_name, name) return None country_code = country["code_cca2"] try: destination = self._destination_matching( country_code=country_code, latitude=latitude, longitude=longitude, address=address, ) except Exception as e: logger.exception(e) return None if not destination: logger.critical( "Bad destination of Hotel(%s) at " "Country(%s) with City(%s)", name, country_name, city, ) return None try: matches = self._hotel_matching( name=name, address=address, longitude=longitude, latitude=latitude, phone=None, wg_destination_id=destination.get("destination_id"), # wg_city_id=destination.get("city_id"), wg_province_id=destination.get("province_id"), wg_country_id=destination.get("country_id"), ) except Empty: logger.critical( "Similarities of Hotel(%s) at Destination(%s) is empty.", name, destination, ) return None except Exception as e: logger.critical( "Falied to got matched hotels. " "Hotel('%s'), Destination(%s)", name, destination, exc_info=e, ) return None for _matched in matches: logger.info( self._set_relationships( _matched["provider"], _matched["oid"], collection_name, str(crawled_hotel["_id"]), )) return True