def setUp(self):
self.test_path = '/test/path'
self.test_cg = 'test/cg'
self.test_msg = cherami_output.PutMessage(data='msg')
self.test_ack_id = 'test_ack_id'
self.logger = config.get_logger('test')
self.test_err_msg = 'test_err_msg'
self.test_delivery_token = (self.test_ack_id, '0:0')
self.output_hosts = mock.Mock(
body=cherami.ReadConsumerGroupHostsResult(hostAddresses=map(
lambda x: cherami.HostAddress(host=str(x), port=x), range(
10))))
self.received_msgs = mock.Mock(
body=cherami_output.ReceiveMessageBatchResult(messages=[
cherami_output.ConsumerMessage(ackId=self.test_ack_id,
payload=self.test_msg)
]))
self.no_msg = mock.Mock(body=cherami_output.ReceiveMessageBatchResult(
messages=[]))
self.ack_ok_response = mock.Mock(body=None)
self.mock_call = mock.Mock()
self.mock_tchannel = mock.Mock()
self.mock_tchannel.thrift.return_value = self.mock_call
def setUp(self):
self.test_path = '/test/path'
self.test_msg = 'test_msg'
self.test_msg_id = 'test_msg_id'
self.test_receipt = 'test_ext_uuid:xxx'
self.test_err_msg = 'test_err_msg'
self.logger = config.get_logger('test')
self.test_crc32 = util.calc_crc(self.test_msg, cherami.ChecksumOption.CRC32IEEE)
self.test_md5 = util.calc_crc(self.test_msg, cherami.ChecksumOption.MD5)
self.publisher_options = mock.Mock(body=cherami.ReadPublisherOptionsResult(
hostProtocols=[cherami.HostProtocol(
protocol=cherami.Protocol.TCHANNEL,
hostAddresses=map(lambda x: cherami.HostAddress(host=str(x), port=x), range(10)))]
))
self.publisher_options_crc32 = mock.Mock(body=cherami.ReadPublisherOptionsResult(
hostProtocols=[cherami.HostProtocol(
protocol=cherami.Protocol.TCHANNEL,
hostAddresses=map(lambda x: cherami.HostAddress(host=str(x), port=x), range(10)))],
checksumOption=cherami.ChecksumOption.CRC32IEEE
))
self.publisher_options_md5 = mock.Mock(body=cherami.ReadPublisherOptionsResult(
hostProtocols=[cherami.HostProtocol(
protocol=cherami.Protocol.TCHANNEL,
hostAddresses=map(lambda x: cherami.HostAddress(host=str(x), port=x), range(10)))],
checksumOption=cherami.ChecksumOption.MD5
))
self.publisher_options_diff = mock.Mock(body=cherami.ReadDestinationHostsResult(
hostProtocols=[cherami.HostProtocol(
protocol=cherami.Protocol.TCHANNEL,
hostAddresses=map(lambda x: cherami.HostAddress(host=str(x), port=x), range(7, 15)))]
))
self.send_ack_success = mock.Mock(body=cherami_input.PutMessageBatchResult(
successMessages=[cherami_input.PutMessageAck(
id=self.test_msg_id,
status=cherami.Status.OK,
receipt=self.test_receipt,
)]
))
self.send_ack_failed = mock.Mock(body=cherami_input.PutMessageBatchResult(
failedMessages=[cherami_input.PutMessageAck(
id=self.test_msg_id,
status=cherami.Status.FAILED,
message=self.test_err_msg,
)]
))
self.mock_call = mock.Mock()
self.mock_tchannel = mock.Mock()
self.mock_tchannel.thrift.return_value = self.mock_call
from __future__ import absolute_import
import mock
import os
import unittest
os.environ['CLAY_CONFIG'] = 'config.json'
from clay import config, mail
log = config.get_logger('clay.tests.mail')
class TestMail(unittest.TestCase):
@mock.patch("smtplib.SMTP")
def test_sendmail(self, mock_SMTP):
mock_SMTP_instance = mock_SMTP.return_value
mailto = '*****@*****.**'
subject = 'This is a subject'
message = 'This is a message'
mail.sendmail(mailto, subject, message)
args, kwargs = mock_SMTP_instance.sendmail.call_args
from_header = config.get('smtp.from')
self.assertEqual(from_header, args[0])
self.assertIn(mailto, args[1])
self.assertIn('To: %s' % mailto, args[2])
self.assertIn('From: %s' % from_header, args[2])
self.assertIn('Subject: %s' % subject, args[2])
self.assertIn('Content-Type: text/html', args[2])
self.assertEqual(len(mock_SMTP_instance.elho.call_args_list), 0)
from __future__ import absolute_import
from __future__ import division
import time
import operator
from datetime import datetime
from dateutil import tz
from clay import config
from cassandra.cluster import Cluster
from conferenceViewer.lib.redshift import Redshift
from instagram.client import InstagramAPI
run_log = config.get_logger('conferenceViewer.run')
class Conference(object):
TOP_10_QUERY = """SELECT * FROM insight.tweet_rankings WHERE yymmdd = '%s' ORDER BY count DESC LIMIT 10;"""
QUERY_TWEETS = """SELECT * FROM insight.conference_details WHERE yymmddhh = '%s'"""
CONFERNECE_LIST_QUERY = """SELECT
DISTINCT
ebe.eb_id as eb_id,
ebe.event_name as event_name,
ebe.event_type as event_type,
ebe.logo_url as logo_url,
ebe.start_time_utc as start_time_utc,
ebe.end_time_utc as end_time_utc,
#!/usr/bin/env python
from __future__ import absolute_import
import sys
from flask import Flask
import werkzeug.serving
from clay import config
log = config.get_logger('clay.server')
def load_middleware(app, name, mwconfig):
log.info('Loading WSGI middleware %s' % name)
try:
modulename, wsgi = name.rsplit('.', 1)
mw = __import__(modulename)
mw = sys.modules[modulename]
mw = getattr(mw, wsgi, None)
if mw is None or not callable(mw):
log.error('No callable named %s in %s (%r)' % (wsgi, modulename, mw))
else:
app = mw(app, **mwconfig)
except Exception:
log.exception('Unable to load WSGI middleware %s' % name)
return app
flask_init = config.get('flask.init', {
'import_name': 'clayapp',
})
from clay import config
from datetime import datetime
from cassandra.cluster import Cluster
script_log = config.get_logger('monkey_job')
class job(object):
TABLE_NAME = "tweet_rankings"
CREATE_TABLE_QUERY = """CREATE TABLE IF NOT EXISTS insight.%s (
yymmdd varchar,
word text,
count int,
PRIMARY KEY ((yymmdd), count)
)
""" % TABLE_NAME
INSERT_DATA_QUERY = """INSERT INTO %s (yymmdd, word, count) VALUES ('%s', '%s', %s)"""
CREATE_LOCATION_TABLE_QUERY = """CREATE TABLE IF NOT EXISTS insight.test_location (
yymmdd varchar,
lat float,
lng float,
conference text,
word text,
count int,
PRIMARY KEY ((yymmdd), lat, lng, conference, count))
)
"""
from __future__ import absolute_import, unicode_literals
import webtest.lint
import webtest
import os
os.environ['CLAY_CONFIG'] = 'config.json'
from clay import app, config, http
import clay.wsgi
log = config.get_logger('clay.tests.hello_world')
# Test application
@app.route('/', methods=['GET'])
@http.cache_control(max_age=3600, public=True, no_cache="Cookies")
def hello_world():
return 'Hello, world!'
# Test methods
app = clay.wsgi.application
app = webtest.lint.middleware(app)
app = webtest.TestApp(app)
def test_hello_world():
res = app.get('/')
assert res.status_int == 200
assert res.body == 'Hello, world!'.encode('utf-8')
from __future__ import absolute_import
import raven.utils.wsgi
import raven
from clay import config
log = config.get_logger("clay.sentry")
client = None
def get_sentry_client():
global client
if client:
return client
dsn = config.get("sentry.url", None)
if not dsn:
return
client = raven.Client(dsn=dsn)
return client
def exception(exc_info, request=None, event_id=None, **extra):
try:
_exception(exc_info, request=request, event_id=event_id, **extra)
except:
log.exception("Unable to send event to sentry")
def _exception(exc_info, request=None, event_id=None, **extra):
client = get_sentry_client()
from clay import config logger = config.get_logger(__name__)
from __future__ import absolute_import
import functools
import socket
import time
from clay import config
log = config.get_logger('clay.stats')
class StatsConnection(object):
'''
Handles the lifecycle of stats sockets and connections.
'''
def __init__(self):
self.sock = None
self.proto = None
self.host = None
self.port = None
self.next_retry = None
self.backoff = 0.5
self.max_backoff = 10.0
def __str__(self):
if self.sock is not None:
return 'StatsConnection %s %s:%i (connected)' % (
self.proto, self.host, self.port)
else:
return 'StatsConnection %s %s:%i (not connected)' % (
self.proto, self.host, self.port)
from __future__ import absolute_import
import raven.utils.wsgi
import raven
from clay import config
log = config.get_logger('clay.sentry')
client = None
def get_sentry_client():
global client
if client:
return client
dsn = config.get('sentry.url', None)
if not dsn:
return
client = raven.Client(dsn=dsn)
return client
def exception(exc_info, request=None, event_id=None, **extra):
try:
_exception(exc_info, request=request, event_id=event_id, **extra)
except:
log.exception('Unable to send event to sentry')
def _exception(exc_info, request=None, event_id=None, **extra):
client = get_sentry_client()
from __future__ import absolute_import
from clay.server import application
from clay import config
log = config.get_logger('clay.wsgi')
views = config.get('views', [])
if not views:
log.warning('No clay view modules configured')
for modulename in views:
log.debug('Loading views from %s' % modulename)
__import__(modulename)
# import zipfile
import sys
import getopt
import subprocess
import traceback
from clay import config
from datetime import datetime
from datamonkey.lib.error import error_email
script_log = config.get_logger('mr_scripts')
def print_help():
print("mr_runner.py: \n--help/-h : help")
print("--script/-s : specify a script, ex: -s 'script_file'")
print("--input/-i: specify a log file")
print("--output-dir: specify output directory")
print("--local/--emr: run mr in local or on aws emr")
print("example: mr_runner -s \"jobs/map_reduce/script.py\" -i \"input_log\" --emr")
class MRJobRunner(object):
def __init__(self, argv):
self.scripts = []
self.output_dir = None
self.loadcache = None
self.env = "local"
self.output, self.error = None, None
from __future__ import absolute_import, unicode_literals
import functools
import socket
import time
from clay import config
log = config.get_logger('clay.stats')
class StatsConnection(object):
'''
Handles the lifecycle of stats sockets and connections.
'''
def __init__(self):
self.sock = None
self.proto = None
self.host = None
self.port = None
self.next_retry = None
self.backoff = 0.5
self.max_backoff = 10.0
def __str__(self):
if self.sock is not None:
return 'StatsConnection %s %s:%i (connected)' % (
self.proto, self.host, self.port)
else:
return 'StatsConnection %s %s:%i (not connected)' % (
self.proto, self.host, self.port)
# import zipfile
import sys
import getopt
import subprocess
import traceback
from clay import config
from datetime import datetime
from datamonkey.lib.error import error_email
script_log = config.get_logger('mr_scripts')
def print_help():
print("mr_runner.py: \n--help/-h : help")
print("--script/-s : specify a script, ex: -s 'script_file'")
print("--input/-i: specify a log file")
print("--output-dir: specify output directory")
print("--local/--emr: run mr in local or on aws emr")
print(
"example: mr_runner -s \"jobs/map_reduce/script.py\" -i \"input_log\" --emr"
)
class MRJobRunner(object):
def __init__(self, argv):
self.scripts = []
self.output_dir = None
self.loadcache = None
self.env = "local"
from __future__ import absolute_import
from celery import Celery
from clay import config
log = config.get_logger('clay.celery')
celery = Celery(log=log)
celery.config_from_object(config.get('celery'))
def main():
'''
Run a celery worker process
'''
celery.worker_main()
if __name__ == '__main__':
main()
from flask import make_response
from collections import namedtuple
import contextlib
import functools
import httplib
import urllib2
import urlparse
import os.path
import ssl
from clay import config
Response = namedtuple("Response", ("status", "headers", "data"))
log = config.get_logger("clay.http")
DEFAULT_CA_CERTS = "/etc/ssl/certs/ca-certificates.crt"
class VerifiedHTTPSOpener(urllib2.HTTPSHandler):
def https_open(self, req):
ca_certs = config.get("http.ca_certs_file", DEFAULT_CA_CERTS)
if config.get("http.verify_server_certificates", True) and os.path.exists(ca_certs):
frags = urlparse.urlparse(req.get_full_url())
ssl.get_server_certificate((frags.hostname, frags.port or 443), ca_certs=ca_certs)
return self.do_open(httplib.HTTPSConnection, req)
urllib2.install_opener(urllib2.build_opener(VerifiedHTTPSOpener))
from __future__ import absolute_import
from celery import Celery
from clay import config
log = config.get_logger('clay.celery')
celery = Celery(log=log)
celery.config_from_object(config.get('celery'))
def main():
'''
Run a celery worker process
'''
celery.worker_main()
if __name__ == '__main__':
main()
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.base import MIMEBase
from email import Encoders
import smtplib
import six
from clay import config
log = config.get_logger('clay.mail')
def _string_or_list(obj):
'''
If obj is a string, it's converted to a single element list, otherwise
it's just returned as-is under the assumption that it's already a list. No
further type checking is performed.
'''
if isinstance(obj, six.string_types):
return [obj]
else:
return obj
def sendmail(mailto, subject, message, subtype='html', charset='utf-8',
smtpconfig=None, attachments={}, use_starttls=False, **headers):
'''
Send an email to the given address. Additional SMTP headers may be specified
as keyword arguments.
'''
import six
try:
from urllib.parse import urlparse
from urllib.request import Request as urllib_Request
from urllib.request import HTTPSHandler, urlopen, install_opener, build_opener
from urllib.error import HTTPError, URLError
except ImportError:
from urlparse import urlparse
from urllib2 import Request as urllib_Request
from urllib2 import urlopen, install_opener, build_opener, HTTPError, HTTPSHandler, URLError
from clay import config
Response = namedtuple('Response', ('status', 'headers', 'data'))
log = config.get_logger('clay.http')
DEFAULT_CA_CERTS = '/etc/ssl/certs/ca-certificates.crt'
class VerifiedHTTPSOpener(HTTPSHandler):
def https_open(self, req):
ca_certs = config.get('http.ca_certs_file', DEFAULT_CA_CERTS)
if config.get('http.verify_server_certificates',
True) and os.path.exists(ca_certs):
frags = urlparse(req.get_full_url())
ssl.get_server_certificate((frags.hostname, frags.port or 443),
ca_certs=ca_certs)
return self.do_open(http.client.HTTPSConnection, req)
from __future__ import absolute_import
import webtest.lint
import webtest
import threading
import os
os.environ['CLAY_CONFIG'] = 'config.json'
from clay import config, database
log = config.get_logger('clay.tests.database')
def test_database_cursor():
with database.read as dbread:
cur = dbread.cursor()
cur.close()
def test_database_create_schema():
with database.write as dbwrite:
cur = dbwrite.cursor()
cur.execute('''
CREATE TABLE IF NOT EXISTS users (
id INTEGER PRIMARY KEY NOT NULL,
email TEXT NOT NULL
)''')
cur.execute('INSERT INTO users(email) VALUES("*****@*****.**")')
cur.close()
dbwrite.commit()
from flask import request
from clay import app, config
import json
log = config.get_logger('clay.docs')
def parse_docstring_param(directive, key, value):
p = {
'name': key,
'description': value.split('{', 1)[0],
'required': False,
'dataType': 'string',
'type': 'primitive',
'allowMultiple': False,
}
if '{' in value and '}' in value:
p.update(json.loads(value[value.find('{'):value.find('}')]))
if directive == 'json':
directive = 'body'
p['type'] = 'complex'
if directive in ('query', 'body', 'path', 'form'):
p['paramType'] = directive
elif directive == 'reqheader':
p['paramType'] = 'header'
else:
log.warning('Ignoring unknown docstring param %s', directive)
return
return p
from __future__ import absolute_import
import webtest.lint
import webtest
import threading
import os
os.environ['CLAY_CONFIG'] = 'config.json'
from clay import config, database
log = config.get_logger('clay.tests.database')
def test_database_cursor():
with database.read as dbread:
cur = dbread.cursor()
cur.close()
def test_database_create_schema():
with database.write as dbwrite:
cur = dbwrite.cursor()
cur.execute('''
CREATE TABLE IF NOT EXISTS users (
id INTEGER PRIMARY KEY NOT NULL,
email TEXT NOT NULL
)''')
cur.execute('INSERT INTO users(email) VALUES("*****@*****.**")')
cur.close()
dbwrite.commit()
with database.read as dbread:
from __future__ import absolute_import, unicode_literals
import unittest
import socket
import os.path
import os
import re
os.environ['CLAY_CONFIG'] = 'config.json'
from clay import config, stats
log = config.get_logger('clay.tests.stats')
class MockTCPListener(object):
def __init__(self, host, port):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.bind((host, port))
self.sock.listen(2)
self.accepted = None
self.buf = ''
def readline(self):
if not self.accepted:
self.accepted, addr = self.sock.accept()
while self.buf.find('\n') == -1:
chunk = self.accepted.recv(1024)
self.buf += chunk.decode()
line, self.buf = self.buf.split('\n', 1)
log.debug('mockserver.readline: %r' % line)
from flask import request
from clay import app, config
import json
log = config.get_logger('clay.docs')
def parse_docstring_param(directive, key, value):
p = {
'name': key,
'description': value.split('{', 1)[0],
'required': False,
'dataType': 'string',
'type': 'primitive',
'allowMultiple': False,
}
if '{' in value and '}' in value:
p.update(json.loads(value[value.find('{'):value.find('}')]))
if directive == 'json':
directive = 'body'
p['type'] = 'complex'
if directive in ('query', 'body', 'path', 'form'):
p['paramType'] = directive
elif directive == 'reqheader':
p['paramType'] = 'header'
else:
log.warning('Ignoring unknown docstring param %s', directive)
return
return p
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.base import MIMEBase
from email import Encoders
import smtplib
import six
from clay import config
log = config.get_logger('clay.mail')
def _string_or_list(obj):
'''
If obj is a string, it's converted to a single element list, otherwise
it's just returned as-is under the assumption that it's already a list. No
further type checking is performed.
'''
if isinstance(obj, six.string_types):
return [obj]
else:
return obj
def sendmail(mailto,
subject,
message,
subtype='html',
charset='utf-8',
smtpconfig=None,
from clay import config
from datetime import datetime
from cassandra.cluster import Cluster
script_log = config.get_logger('monkey_job')
class job(object):
TABLE_NAME = "tweet_rankings"
CREATE_TABLE_QUERY = """CREATE TABLE IF NOT EXISTS insight.%s (
yymmdd varchar,
word text,
count int,
PRIMARY KEY ((yymmdd), count)
)
""" % TABLE_NAME
INSERT_DATA_QUERY = """INSERT INTO %s (yymmdd, word, count) VALUES ('%s', '%s', %s)"""
CREATE_LOCATION_TABLE_QUERY = """CREATE TABLE IF NOT EXISTS insight.test_location (
yymmdd varchar,
lat float,
lng float,
conference text,
word text,
count int,
PRIMARY KEY ((yymmdd), lat, lng, conference, count))
)
"""
from __future__ import absolute_import
import mock
import os
import unittest
os.environ['CLAY_CONFIG'] = 'config.json'
from clay import config, mail
log = config.get_logger('clay.tests.mail')
class TestMail(unittest.TestCase):
@mock.patch("smtplib.SMTP")
def test_sendmail(self, mock_SMTP):
mock_SMTP_instance = mock_SMTP.return_value
mailto = '*****@*****.**'
subject = 'This is a subject'
message = 'This is a message'
mail.sendmail(mailto, subject, message)
args, kwargs = mock_SMTP_instance.sendmail.call_args
from_header = config.get('smtp.from')
self.assertEqual(from_header, args[0])
self.assertIn(mailto, args[1])
self.assertIn('To: %s' % mailto, args[2])
self.assertIn('From: %s' % from_header, args[2])
self.assertIn('Subject: %s' % subject, args[2])
self.assertIn('Content-Type: text/html', args[2])
from __future__ import absolute_import
from __future__ import division
import time
import operator
from datetime import datetime
from dateutil import tz
from clay import config
from cassandra.cluster import Cluster
from conferenceViewer.lib.redshift import Redshift
from instagram.client import InstagramAPI
run_log = config.get_logger('conferenceViewer.run')
class Conference(object):
TOP_10_QUERY = """SELECT * FROM insight.tweet_rankings WHERE yymmdd = '%s' ORDER BY count DESC LIMIT 10;"""
QUERY_TWEETS = """SELECT * FROM insight.conference_details WHERE yymmddhh = '%s'"""
CONFERNECE_LIST_QUERY = """SELECT
DISTINCT
ebe.eb_id as eb_id,
ebe.event_name as event_name,
ebe.event_type as event_type,
ebe.logo_url as logo_url,
ebe.start_time_utc as start_time_utc,
ebe.end_time_utc as end_time_utc,
from __future__ import absolute_import
import webtest.lint
import webtest
import os
os.environ['CLAY_CONFIG'] = 'config.json'
from clay import app, config, http
import clay.wsgi
log = config.get_logger('clay.tests.hello_world')
# Test application
@app.route('/', methods=['GET'])
@http.cache_control(max_age=3600, public=True, no_cache="Cookies")
def hello_world():
return 'Hello, world!'
# Test methods
app = clay.wsgi.application
app = webtest.lint.middleware(app)
app = webtest.TestApp(app)
def test_hello_world():
res = app.get('/')
assert res.status_int == 200
assert res.body == 'Hello, world!'
def test_cache_control():
from flask import make_response
from collections import namedtuple
import contextlib
import functools
import httplib
import urllib2
import urlparse
import os.path
import ssl
import six
from clay import config
Response = namedtuple('Response', ('status', 'headers', 'data'))
log = config.get_logger('clay.http')
DEFAULT_CA_CERTS = '/etc/ssl/certs/ca-certificates.crt'
class VerifiedHTTPSOpener(urllib2.HTTPSHandler):
def https_open(self, req):
ca_certs = config.get('http.ca_certs_file', DEFAULT_CA_CERTS)
if config.get('http.verify_server_certificates', True) and os.path.exists(ca_certs):
frags = urlparse.urlparse(req.get_full_url())
ssl.get_server_certificate(
(frags.hostname, frags.port or 443),
ca_certs=ca_certs
)
return self.do_open(httplib.HTTPSConnection, req)
from __future__ import absolute_import
import unittest
import socket
import os.path
import os
import re
os.environ['CLAY_CONFIG'] = 'config.json'
from clay import config, stats
log = config.get_logger('clay.tests.stats')
class MockTCPListener(object):
def __init__(self, host, port):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.bind((host, port))
self.sock.listen(2)
self.accepted = None
self.buf = ''
def readline(self):
if not self.accepted:
self.accepted, addr = self.sock.accept()
while self.buf.find('\n') == -1:
chunk = self.accepted.recv(1024)
self.buf += chunk
line, self.buf = self.buf.split('\n', 1)
log.debug('mockserver.readline: %r' % line)