def remove_scam_emails(): scam_emails = get_latest_scam_emails() for scam_emails_chunk in chunks(scam_emails, 100): query = Office.query.filter(Office.email.in_(scam_emails_chunk)) office_count = query.count() if office_count: query.update({Office.email: ''}, synchronize_session="fetch") db_session.commit() logger.info( "Removed a chunk of %d scam emails from %d offices.", len(scam_emails_chunk), office_count, )
def delete_deletable_offices(self): con, cur = import_util.create_cursor() if self.deletable_sirets: for sirets in chunks(list(self.deletable_sirets), 500): stringified_siret_list = ",".join(sirets) logger.info("deleting a chunk of %i offices...", len(sirets)) query = """DELETE FROM %s where siret IN (%s)""" % (settings.RAW_OFFICE_TABLE, stringified_siret_list) try: cur.execute(query) con.commit() except: logger.warning("error while deleting chunk of sirets : %s", sirets) raise cur.close() con.close() logger.info("%i no longer existing offices deleted.", len(self.deletable_sirets))
def remove_scam_emails(): scam_emails = get_latest_scam_emails() for scam_emails_chunk in chunks(scam_emails, 100): query = Office.query.filter(Office.email.in_(scam_emails_chunk)) office_count = query.count() if office_count: history = [] for office in query.all(): history.append( HistoryBlacklist(email=office.email, datetime_removal=datetime.datetime.now())) db_session.add_all(history) query.update({Office.email: ''}, synchronize_session="fetch") db_session.commit() logger.info( "Removed a chunk of %d scam emails from %d offices.", len(scam_emails_chunk), office_count, )
def get_offers_for_romes(self, romes): offers = [] for romes_batch in chunks(romes, OFFRES_ESD_MAXIMUM_ROMES): url = OFFRES_ESD_ENDPOINT_URL params = { 'range': "0-{}".format(OFFRES_ESD_MAXIMUM_PAGE_SIZE - 1), 'sort': 1, 'codeROME': ",".join(romes_batch), 'natureContrat': ",".join(self.get_contract_nature_codes()), 'commune': self.commune_id, 'distance': min(self.distance, OFFRES_ESD_MAXIMUM_DISTANCE), } response = esd.get_response(url, params) # Convenient reminder to dump json to file for test mockups. # json.dump(response, json_file, sort_keys=True, indent=4) offers += response['resultats'] return offers