import json
from job_update_api import update_job_status
from sql_data import db, connect_db
import csv
from sheet import updateAction, updateLink
import requests
import ftplib
from custom_events.main import execute_pipeline
from configuration import createLogger
cLogger = createLogger("external_setup")
def upload_file(filename, path):
ftp = ftplib.FTP("jlabs.co")
ftp.login("jlab", "coldplay")
ftp.cwd("/jlabs_co/wadi/query_results")
ftp.storlines("STOR "+filename, open(path))
ftp.close()
def save_to_file(lst_obj, filename, headers):
print "Query executed, writing file"
with open(filename, 'w') as cfile:
writer = csv.writer(cfile, quoting=csv.QUOTE_ALL)
writer.writerow(headers)
writer.writerows(lst_obj)
def work_external_data(event):
try:
oid = event['oid']
from sheet import updateAction, get_testing_sheet, get_custom_sheet, get_block_sheet, getFileLink
from sql_data import db
import json
import string
import requests
import csv
from configuration import createLogger
QUERRY = {
"all" : "select distinct b.number,if(a.fk_language=1,'English','Arabic') as language from customer a inner join customer_phone b on b.fk_customer = a.id_customer order by a.id_customer desc",
"all_uae" : "select distinct b.number,if(a.fk_language=1,'English','Arabic') as language from customer a inner join customer_phone b on b.fk_customer = a.id_customer where a.fk_country = 3 order by a.id_customer desc",
"all_ksa" : "select distinct b.number,if(a.fk_language=1,'English','Arabic') as language from customer a inner join customer_phone b on b.fk_customer = a.id_customer where a.fk_country = 193 order by a.id_customer desc",
"other" : "select distinct phone,if(language_code='en','English','Arabic') from promotion_subscription WHERE promotion_type LIKE %s"
}
cLogger = createLogger("data_loader")
# ------------ Helper functions ------------------
def str_to_hex(text):
text = text.strip("u").strip("'")
arabic_hex = [hex(ord(b)).replace("x","").upper().zfill(4) for b in text]
arabic_hex.append("000A")
text_update = "".join(arabic_hex)
return text_update
def clean_english(text):
# Support for en and em dash which is a common english unicode
repText = ''.join(map(
lambda c: '-' if c == u'\u2013' or c == u'\u2014' else c,
text
))
FROM customer a INNER JOIN customer_phone b ON b.fk_customer = a.id_customer
ORDER BY a.id_customer DESC""",
"all_uae": """SELECT DISTINCT b.number,if(a.fk_language=1,'English','Arabic') AS LANGUAGE
FROM customer a INNER JOIN customer_phone b ON b.fk_customer = a.id_customer
WHERE a.fk_country = 3
ORDER BY a.id_customer DESC""",
"all_ksa": """SELECT DISTINCT b.number,if(a.fk_language=1,'English','Arabic') AS language
FROM customer a INNER JOIN customer_phone b ON b.fk_customer = a.id_customer
WHERE a.fk_country = 193
ORDER BY a.id_customer DESC""",
"other": """SELECT DISTINCT phone,if(language_code='en','English','Arabic')
FROM promotion_subscription
WHERE promotion_type LIKE %s"""
}
cLogger = createLogger("data_loader")
# ---------------------------------------------------- Helper functions ------------------------------------------------
def str_to_hex(text):
""" Converts a string to hex values, used for arabic messages """
text = text.strip("u").strip("'")
arabic_hex = [hex(ord(b)).replace("x", "").upper().zfill(4) for b in text]
arabic_hex.append("000A")
text_update = "".join(arabic_hex)
return text_update
def clean_english(text):
""" Converts any possible unicode to its ascii variant else removes unicode from english message """
# Support for en and em dash which is a common english unicode
repText = ''.join(map(
import json
from job_update_api import update_job_status
from sql_data import db, connect_db
import csv
from sheet import updateAction, updateLink
import requests
import ftplib
from custom_events.main import execute_pipeline
from configuration import createLogger
cLogger = createLogger("external_setup")
def upload_file(filename, path):
ftp = ftplib.FTP("jlabs.co")
ftp.login("jlab", "coldplay")
ftp.cwd("/jlabs_co/wadi/query_results")
ftp.storlines("STOR " + filename, open(path))
ftp.close()
def save_to_file(lst_obj, filename, headers):
print "Query executed, writing file"
with open(filename, 'w') as cfile:
writer = csv.writer(cfile, quoting=csv.QUOTE_ALL)
writer.writerow(headers)
writer.writerows(lst_obj)
def work_external_data(event):
try:
import httplib2
# Do OAuth2 stuff to create credentials object
from oauth2client.file import Storage
from oauth2client.client import flow_from_clientsecrets
from oauth2client import tools
import gspread
import requests
from job_update_api import update_job_status
from configuration import createLogger
import os
cLogger = createLogger("sheet")
_cache = {} # id against sheet row numbers
def local_file(name):
cdir = os.path.dirname(__file__)
filename = os.path.join(cdir, name)
return filename
def get_worksheet(i):
storage = Storage(local_file("creds.data"))
credentials = storage.get()
if credentials is None or credentials.invalid:
flags = tools.argparser.parse_args(args=[])
flow = flow_from_clientsecrets(
local_file("client_secret.json"),
scope=["https://spreadsheets.google.com/feeds"])
credentials = tools.run_flow(flow, storage, flags)
if credentials.access_token_expired:
# Main application instance, which will be the producer for the rabbitMQ
from data_loader import load_data
from sheet import updateAction
import json
import pika
from configuration import config, createLogger
from external_setup import work_external_data
cLogger = createLogger("handler")
def watcher(event, send_sms, log):
print "Debug: Watcher started ..."
try:
## Full code for main parent
if event['type'] == 'send_sms':
data = event['data']
res, payloadArr = load_data(data)
if not res:
raise Exception
log(data['id'], len(payloadArr) - 1) # The last one is the sentinel
print "Payload array size", len(payloadArr) - 1
for payload in payloadArr:
send_sms(payload)
elif event['type'] == 'external_setup':
data = event['data']
result = work_external_data(data)
print "DEBUG, external event, loaded = "+str(result)
if not result:
updateAction(data['id'], 'Data Load Failed', event.get('oid'))
