def getTask(DB_CONF_FILE,
DSC_PROCESS,
COD_INTEGRATION,
COD_COUNTRY,
base_api_token,
cod_task=999999999):
client = basecrm.Client(access_token=base_api_token)
number_of_errors = 0
keep_trying = True
while keep_trying:
try:
if (number_of_errors > MAX_ERRORS_SKIPPED):
return -2
getBaseTask = client.tasks.retrieve(id=cod_task)
break
except basecrm.errors.ServerError as err:
scai.logError(DB_CONF_FILE, DSC_PROCESS, COD_INTEGRATION,
COD_COUNTRY, "basecrm.errors.ServerError", str(err))
print("Error: basecrm.errors.ServerError\nDescription: " +
str(err) + "\nTrying again...")
number_of_errors = number_of_errors + 1
except basecrm.errors.RateLimitError as err:
scai.logError(DB_CONF_FILE, DSC_PROCESS, COD_INTEGRATION,
COD_COUNTRY, "basecrm.errors.RateLimitError",
str(err))
print("Error: basecrm.errors.RateLimitError\nDescription: " +
str(err) + "\nTrying again in 1 second...")
number_of_errors = number_of_errors + 1
time.sleep(1)
except requests.exceptions.ConnectionError as err:
scai.logError(DB_CONF_FILE, DSC_PROCESS, COD_INTEGRATION,
COD_COUNTRY, "requests.exceptions.ConnectionError",
str(err))
print("Error: requests.exceptions.ConnectionError\nDescription: " +
str(err) + "\nTrying again in 1 second...")
number_of_errors = number_of_errors + 1
time.sleep(1)
except basecrm.errors.RequestError as err:
for error in err.errors:
#print (error.code)
if (error.code == "not_found"):
return -1
scai.logError(DB_CONF_FILE, DSC_PROCESS, COD_INTEGRATION,
COD_COUNTRY, "basecrm.errors.ServerError", str(err))
print("Error: basecrm.errors.ServerError\nDescription: " +
str(err) + "\nTrying again...")
number_of_errors = number_of_errors + 1
except Exception as err:
scai.logError(DB_CONF_FILE, DSC_PROCESS, COD_INTEGRATION,
COD_COUNTRY, "Exception with general handling",
str(err))
print("Error\nDescription: " + str(err) +
"\nTrying again in 1 second...")
number_of_errors = number_of_errors + 1
time.sleep(1)
return getBaseTask
def test_connection_error(self):
http_client = basecrm.Client(access_token='X'*64,
base_url='https://somethingthatnotexist.fail',
user_agent=self.user_agent,
verbose=True).http_client
try:
http_client.get("/users/self")
except ConnectionError as e:
# OK
self.assertTrue(True)
except Exception as e:
self.fail("Unexpected exception type %r" % type(e))
else:
self.fail("Exception expected")
def test_authentication_error(self):
http_client = basecrm.Client(access_token='X'*64,
user_agent=self.user_agent,
verbose=True).http_client
try:
http_client.get("/users/self")
except basecrm.RequestError as exception:
self.assertEqual(exception.http_status, 401)
self.assertGreaterEqual(len(exception.logref), 1)
self.assertIsInstance(exception.errors, list)
self.assertGreaterEqual(len(exception.errors), 1)
except Exception as e:
self.fail("Unexpected exception type %r" % type(e))
else:
self.fail("Exception expected")
def main(conf_file):
base_api_token = json.load(open(conf_file))['base_api_token_stvpt']
schema = json.load(open(conf_file))['schema']
country = json.load(open(conf_file))['country_pt']
category = json.load(open(conf_file))['category_stvpt']
data_path = json.load(open(conf_file))['data_path']
bucketName = json.load(open(conf_file))['bucketName']
manifest_path = json.load(open(conf_file))['manifest_path']
client = basecrm.Client(access_token=base_api_token)
date = str(datetime.now().strftime('%Y/%m/%d/'))
### TODO - DELETE S3 PATH BEFORE UNLOADING!!!!
deletePreviousS3Files(bucketName, data_path)
s3_fulldump_contacts(client,bucketName,data_path,category,country)
loadFromS3toRedshift(conf_file,schema,category,country,bucketName,data_path,date,manifest_path)
def getDatabaseConnection(conf_file):
data = json.load(open(conf_file))
return psycopg2.connect(dbname=data['dbname'],
host=data['host'],
port=data['port'],
user=data['user'],
password=data['pass'])
conf_file = sys.argv[1] # File with source database
base_api_token = json.load(open(conf_file))['base_api_token_stvpt']
client = basecrm.Client(access_token=base_api_token)
conn = getDatabaseConnection(conf_file)
cur = conn.cursor()
print('Start Truncate aut_stvpt_base_to_bd_contact: ' +
time.strftime("%H:%M:%S"))
cur.execute(
"truncate table crm_integration_anlt.aut_stvpt_base_to_bd_contact; ")
print('End Truncate aut_stvpt_base_to_bd_contact: ' +
time.strftime("%H:%M:%S"))
print('Start Truncate aut_stvpt_base_to_bd_deal: ' + time.strftime("%H:%M:%S"))
cur.execute("truncate table crm_integration_anlt.aut_stvpt_base_to_bd_deal; ")
print('End Truncate aut_stvpt_base_to_bd_deal: ' + time.strftime("%H:%M:%S"))
def createTask(DB_CONF_FILE,
DSC_PROCESS,
COD_INTEGRATION,
COD_COUNTRY,
base_api_token,
content,
resource_type,
due_date='2099-12-31 00:00:00',
owner_id=1,
resource_id=999999999,
completed=False,
remind_at='2099-12-31 00:00:00'):
client = basecrm.Client(access_token=base_api_token)
number_of_errors = 0
keep_trying = True
while keep_trying:
try:
if (number_of_errors > MAX_ERRORS_SKIPPED):
return -2
task = client.tasks.create(content=content,
due_date=due_date,
owner_id=owner_id,
resource_type=resource_type,
resource_id=resource_id,
completed=completed,
remind_at=remind_at)
break
except basecrm.errors.ServerError as err:
scai.logError(DB_CONF_FILE, DSC_PROCESS, COD_INTEGRATION,
COD_COUNTRY, "basecrm.errors.ServerError", str(err))
print("Error: basecrm.errors.ServerError\nDescription: " +
str(err) + "\nTrying again...")
number_of_errors = number_of_errors + 1
except basecrm.errors.RateLimitError as err:
scai.logError(DB_CONF_FILE, DSC_PROCESS, COD_INTEGRATION,
COD_COUNTRY, "basecrm.errors.RateLimitError",
str(err))
print("Error: basecrm.errors.RateLimitError\nDescription: " +
str(err) + "\nTrying again in 1 second...")
number_of_errors = number_of_errors + 1
time.sleep(1)
except requests.exceptions.ConnectionError as err:
scai.logError(DB_CONF_FILE, DSC_PROCESS, COD_INTEGRATION,
COD_COUNTRY, "requests.exceptions.ConnectionError",
str(err))
print("Error: requests.exceptions.ConnectionError\nDescription: " +
str(err) + "\nTrying again in 1 second...")
number_of_errors = number_of_errors + 1
time.sleep(1)
except Exception as err:
scai.logError(DB_CONF_FILE, DSC_PROCESS, COD_INTEGRATION,
COD_COUNTRY, "Exception with general handling",
str(err))
print("Error\nDescription: " + str(err) +
"\nTrying again in 1 second...")
number_of_errors = number_of_errors + 1
time.sleep(1)
return task.id
def client(self):
return basecrm.Client(access_token=self.access_token,
base_url=self.base_url,
user_agent=self.user_agent,
verbose=True)
def get_client():
return basecrm.Client(access_token=settings.BASECRM_TOKEN)
from datetime import date, datetime
from fulldump_resources import *
print(datetime.now().time())
conf_file = sys.argv[1]
##################################################
#Get access token and other data
##################################################
data = json.load(open(conf_file))
access_token_base = data['base_api_token']
bucketName = data['bucket_name']
path = data['s3_data_path']
category = data['category']
country = data['country']
resources = data['resources'].split(',')
##################################################
#Configure Base API client
##################################################
client = basecrm.Client(access_token=access_token_base)
##################################################
# Full dumps
##################################################
for resource in resources:
mapping_fulldump_methods(resource, access_token_base, bucketName, path,
client, country, category)
print(datetime.now().time())
def getBaseConnection():
client = basecrm.Client(access_token=BASE_API_TOKEN)
return client
def main(db_conf_file, conf_file):
print(datetime.now().time())
data = json.load(open(conf_file))
cod_source_system = data['cod_source_system']
base_api_token = data['base_api_token']
global DSC_PROCESS
DSC_PROCESS = data['dsc_process']
global COD_COUNTRY
COD_COUNTRY = int(data['cod_country'])
# Create Redshift Connection
print('Connecting to Database...')
conn = getDatabaseConnection(db_conf_file)
cur = conn.cursor()
scai.processStart(db_conf_file, DSC_PROCESS, COD_INTEGRATION,
COD_COUNTRY) # SCAI
# Obtain the list of custom fields and contacts to update in Base; This is a list of tuples (opr_contact, dsc_custom_field, custom_field_value)
print(
'Querying for contacts with custom fields to update to Base with cod_source_system '
+ cod_source_system + '...')
# TODO: Confirm dsc_process_short name
try:
cur.execute(
"SELECT cast(contact.opr_contact as integer), "\
" custom_field.dsc_custom_field, "\
" fac.custom_field_value "\
"FROM crm_integration_anlt.t_fac_base_integration_snap fac, "\
"crm_integration_anlt.t_lkp_contact contact, "\
"crm_integration_anlt.t_lkp_custom_field custom_field, "\
"crm_integration_anlt.t_rel_scai_integration_process rel, "\
"crm_integration_anlt.t_lkp_scai_process process "\
"WHERE fac.cod_custom_field = custom_field.cod_custom_field "\
"AND fac.cod_contact = contact.cod_contact "\
"AND fac.dat_snap = rel.dat_processing "\
"AND rel.cod_process = process.cod_process "\
"AND process.dsc_process_short = '" + DSC_PROCESS + "' "\
"AND fac.cod_source_system = " + cod_source_system + " "\
"AND contact.valid_to = 20991231;")
except Exception as e:
scai.processEnd(db_conf_file, DSC_PROCESS, COD_INTEGRATION,
COD_COUNTRY, '', '', 3) # SCAI
scai.integrationEnd(db_conf_file, COD_INTEGRATION, COD_COUNTRY,
3) # SCAI
print(e)
print(e.pgerror)
sys.exit("The process aborted with error.")
print('Extracting query results...')
result_list = cur.fetchall()
#print('Results:')
#print(result_list)
print('Closing Database connection...')
cur.close()
conn.close()
print(datetime.now().time())
# Create Base Connection
print('Connecting to ' + DSC_PROCESS + '...')
client = basecrm.Client(access_token=base_api_token)
# Put all query results in a dictionary with key as opr_contact, and value as a list of the tuples the query returned (one for each custom field of that contact)
result_dictionary = dict()
for result in result_list:
if result[CONTACT_ID_IDX] in result_dictionary:
result_dictionary[result[CONTACT_ID_IDX]].append(result)
else:
result_dictionary[result[CONTACT_ID_IDX]] = [result]
# Get contacts and iterate through them
contact_dictionary = dict()
page_nbr = 1
contacts_data = '1'
number_of_updates = 0
number_of_errors = 0
while len(contacts_data) > 0:
while True:
try:
if (number_of_errors > MAX_ERRORS_SKIPPED):
scai.processEnd(db_conf_file, DSC_PROCESS, COD_INTEGRATION,
COD_COUNTRY, '', '', 3) # SCAI
scai.integrationEnd(db_conf_file, COD_INTEGRATION,
COD_COUNTRY, 3) # SCAI
sys.exit("The process aborted for exceeding " +
str(MAX_ERRORS_SKIPPED) + " errors.")
print('Page #' + str(page_nbr))
contacts_data = client.contacts.list(page=page_nbr,
per_page=100)
break
except basecrm.errors.ServerError as err:
scai.logError(db_conf_file, DSC_PROCESS, COD_INTEGRATION,
COD_COUNTRY, "basecrm.errors.ServerError",
str(err))
print("Error: basecrm.errors.ServerError\nDescription: " +
str(err) + "\nTrying again...")
number_of_errors = number_of_errors + 1
except basecrm.errors.RateLimitError as err:
scai.logError(db_conf_file, DSC_PROCESS, COD_INTEGRATION,
COD_COUNTRY, "basecrm.errors.RateLimitError",
str(err))
print("Error: basecrm.errors.RateLimitError\nDescription: " +
str(err) + "\nTrying again in 1 second...")
number_of_errors = number_of_errors + 1
time.sleep(1)
except requests.exceptions.ConnectionError as err:
scai.logError(db_conf_file, DSC_PROCESS, COD_INTEGRATION,
COD_COUNTRY,
"requests.exceptions.ConnectionError", str(err))
print(
"Error: requests.exceptions.ConnectionError\nDescription: "
+ str(err) + "\nTrying again in 1 second...")
number_of_errors = number_of_errors + 1
time.sleep(1)
except Exception as err:
scai.logError(db_conf_file, DSC_PROCESS, COD_INTEGRATION,
COD_COUNTRY, "Exception with general handling",
str(err))
print("Error\nDescription: " + str(err) +
"\nTrying again in 1 second...")
number_of_errors = number_of_errors + 1
time.sleep(1)
# Code could be further improved if all contacts are acquired from Base first, put in a dictionary, and then do updates later (will use much more memory, however)
# Alternatively, a dictionary with 100 contacts could be created for every iteration here, instead of putting all contacts in a single dictionary; there could still be some time gains compared to current implementation
for contact in contacts_data:
if contact.id in result_dictionary:
for result in result_dictionary[contact.id]:
contact.custom_fields[
result[CUSTOM_FIELD_NAME_IDX]] = result[
CUSTOM_FIELD_VALUE_IDX]
number_of_updates = number_of_updates + 1
contact_dictionary[contact.id] = contact
page_nbr = page_nbr + 1
print(datetime.now().time())
print('Number of updates done in code: ' + str(number_of_updates))
# Update contacts in Base
print('Updating #' + str(len(contact_dictionary)) + ' contacts in Base')
#input('Ready to send contacts to Base. Proceed?')
# Threading implementation
number_active_threads = 0
contact_list = list(contact_dictionary.values())
number_contacts = len(contact_list)
contacts_per_thread = -(-number_contacts // MAX_ACTIVE_THREADS
) # Ceiling of integer division
thread_list = []
thread_return_values_queue = queue.Queue()
i = 0
j = contacts_per_thread
for n in range(0, MAX_ACTIVE_THREADS):
t = threading.Thread(target=updateContactsInBase,
args=(client, contact_list[i:j], db_conf_file,
thread_return_values_queue))
thread_list.append(t)
t.start()
print('Spawned thread #' + str(n + 1))
i = i + contacts_per_thread
j = j + contacts_per_thread
if j > number_contacts:
j = number_contacts
for t in thread_list:
t.join()
number_of_updates = 0
while not thread_return_values_queue.empty():
number_of_updates = number_of_updates + thread_return_values_queue.get(
)
print('Number of updates done in ' + DSC_PROCESS + ': ' +
str(number_of_updates))
scai.processEnd(db_conf_file, DSC_PROCESS, COD_INTEGRATION, COD_COUNTRY,
'', '', 1) # SCAI
print(datetime.now().time())
print('Done\n')
