import json from job_update_api import update_job_status from sql_data import db, connect_db import csv from sheet import updateAction, updateLink import requests import ftplib from custom_events.main import execute_pipeline from configuration import createLogger cLogger = createLogger("external_setup") def upload_file(filename, path): ftp = ftplib.FTP("jlabs.co") ftp.login("jlab", "coldplay") ftp.cwd("/jlabs_co/wadi/query_results") ftp.storlines("STOR "+filename, open(path)) ftp.close() def save_to_file(lst_obj, filename, headers): print "Query executed, writing file" with open(filename, 'w') as cfile: writer = csv.writer(cfile, quoting=csv.QUOTE_ALL) writer.writerow(headers) writer.writerows(lst_obj) def work_external_data(event): try: oid = event['oid']
from sheet import updateAction, get_testing_sheet, get_custom_sheet, get_block_sheet, getFileLink from sql_data import db import json import string import requests import csv from configuration import createLogger QUERRY = { "all" : "select distinct b.number,if(a.fk_language=1,'English','Arabic') as language from customer a inner join customer_phone b on b.fk_customer = a.id_customer order by a.id_customer desc", "all_uae" : "select distinct b.number,if(a.fk_language=1,'English','Arabic') as language from customer a inner join customer_phone b on b.fk_customer = a.id_customer where a.fk_country = 3 order by a.id_customer desc", "all_ksa" : "select distinct b.number,if(a.fk_language=1,'English','Arabic') as language from customer a inner join customer_phone b on b.fk_customer = a.id_customer where a.fk_country = 193 order by a.id_customer desc", "other" : "select distinct phone,if(language_code='en','English','Arabic') from promotion_subscription WHERE promotion_type LIKE %s" } cLogger = createLogger("data_loader") # ------------ Helper functions ------------------ def str_to_hex(text): text = text.strip("u").strip("'") arabic_hex = [hex(ord(b)).replace("x","").upper().zfill(4) for b in text] arabic_hex.append("000A") text_update = "".join(arabic_hex) return text_update def clean_english(text): # Support for en and em dash which is a common english unicode repText = ''.join(map( lambda c: '-' if c == u'\u2013' or c == u'\u2014' else c, text ))
FROM customer a INNER JOIN customer_phone b ON b.fk_customer = a.id_customer ORDER BY a.id_customer DESC""", "all_uae": """SELECT DISTINCT b.number,if(a.fk_language=1,'English','Arabic') AS LANGUAGE FROM customer a INNER JOIN customer_phone b ON b.fk_customer = a.id_customer WHERE a.fk_country = 3 ORDER BY a.id_customer DESC""", "all_ksa": """SELECT DISTINCT b.number,if(a.fk_language=1,'English','Arabic') AS language FROM customer a INNER JOIN customer_phone b ON b.fk_customer = a.id_customer WHERE a.fk_country = 193 ORDER BY a.id_customer DESC""", "other": """SELECT DISTINCT phone,if(language_code='en','English','Arabic') FROM promotion_subscription WHERE promotion_type LIKE %s""" } cLogger = createLogger("data_loader") # ---------------------------------------------------- Helper functions ------------------------------------------------ def str_to_hex(text): """ Converts a string to hex values, used for arabic messages """ text = text.strip("u").strip("'") arabic_hex = [hex(ord(b)).replace("x", "").upper().zfill(4) for b in text] arabic_hex.append("000A") text_update = "".join(arabic_hex) return text_update def clean_english(text): """ Converts any possible unicode to its ascii variant else removes unicode from english message """ # Support for en and em dash which is a common english unicode repText = ''.join(map(
import json from job_update_api import update_job_status from sql_data import db, connect_db import csv from sheet import updateAction, updateLink import requests import ftplib from custom_events.main import execute_pipeline from configuration import createLogger cLogger = createLogger("external_setup") def upload_file(filename, path): ftp = ftplib.FTP("jlabs.co") ftp.login("jlab", "coldplay") ftp.cwd("/jlabs_co/wadi/query_results") ftp.storlines("STOR " + filename, open(path)) ftp.close() def save_to_file(lst_obj, filename, headers): print "Query executed, writing file" with open(filename, 'w') as cfile: writer = csv.writer(cfile, quoting=csv.QUOTE_ALL) writer.writerow(headers) writer.writerows(lst_obj) def work_external_data(event): try:
import httplib2 # Do OAuth2 stuff to create credentials object from oauth2client.file import Storage from oauth2client.client import flow_from_clientsecrets from oauth2client import tools import gspread import requests from job_update_api import update_job_status from configuration import createLogger import os cLogger = createLogger("sheet") _cache = {} # id against sheet row numbers def local_file(name): cdir = os.path.dirname(__file__) filename = os.path.join(cdir, name) return filename def get_worksheet(i): storage = Storage(local_file("creds.data")) credentials = storage.get() if credentials is None or credentials.invalid: flags = tools.argparser.parse_args(args=[]) flow = flow_from_clientsecrets( local_file("client_secret.json"), scope=["https://spreadsheets.google.com/feeds"]) credentials = tools.run_flow(flow, storage, flags) if credentials.access_token_expired:
# Main application instance, which will be the producer for the rabbitMQ from data_loader import load_data from sheet import updateAction import json import pika from configuration import config, createLogger from external_setup import work_external_data cLogger = createLogger("handler") def watcher(event, send_sms, log): print "Debug: Watcher started ..." try: ## Full code for main parent if event['type'] == 'send_sms': data = event['data'] res, payloadArr = load_data(data) if not res: raise Exception log(data['id'], len(payloadArr) - 1) # The last one is the sentinel print "Payload array size", len(payloadArr) - 1 for payload in payloadArr: send_sms(payload) elif event['type'] == 'external_setup': data = event['data'] result = work_external_data(data) print "DEBUG, external event, loaded = "+str(result) if not result: updateAction(data['id'], 'Data Load Failed', event.get('oid'))