def main():
global log
ap = args.get_parser()
ap.add_argument("--s_date", type=str, help="the start date to ingest: format mmddyyyy")
ap.add_argument("--e_date", type=str, help="the end of date to ingest: format mmddyyyy")
ap.add_argument("--o", type=str, help="the output directory")
ap.add_argument("--region", type=str, help="the region of the web site")
arg = ap.parse_args()
logs.init(arg)
t_format = "%m%d%Y"
s_date = datetime.strptime(arg.s_date, t_format)
e_date = datetime.strptime(arg.e_date, t_format)
d_delta = (e_date - s_date).days
seen_it = shelve.open("%s_reuters_news_seen_it.db" % (arg.region))
i = 0
while i <= d_delta:
day_str = datetime.strftime(s_date + timedelta(days=i), t_format)
print "Extracting %s" % (day_str)
"write news to day file"
with open("%s%s%s_ita_reuters_%s.txt" % (arg.o, os.sep, day_str, arg.region), "w") as w:
daily_news = get_daily_news(day_str, seen_it, arg.region)
for news in daily_news:
w.write(json.dumps(news) + "\n")
i += 1
def main():
ap = args.get_parser()
ap.add_argument('--rev_file', metavar='FILE', type=str, required=False,
help='File which stores last revision id.', default="last_rev_file.txt")
arg = ap.parse_args()
logs.init(arg)
log.info("Run Started")
api = API()
recent_changes = api.get_recent_changes()
# Filter to only revisions newer than last run
last_rev_file = arg.rev_file
if os.path.exists(last_rev_file):
with open(last_rev_file) as f:
last_rev = int(f.read())
recent_changes = [change for change in recent_changes if change['revid'] > last_rev]
recently_changed_page_ids = set(str(change['pageid']) for change in recent_changes)
latest_revisions = api.get_latest_revision(recently_changed_page_ids)
print(len(latest_revisions))
print(latest_revisions)
# TODO actually do something with revisions
with open(last_rev_file, mode='w') as f:
f.write(str(max([int(page['revisions'][0]['revid']) for page in latest_revisions.values()])))
def main():
"""
Utility to cache messages from all queues from the --hostname provided with 'cache: true' option set in embers.conf
--hostname : Cache all active queues on this host
--log_file : Path to write the log file to
--log_level : Logging level
"""
from etool import args
global log
arg_parser = args.get_parser()
arg_parser.add_argument('--hostname', metavar='HOSTNAME', type=str, default=environ.get('HOSTNAME', None),
help="The hostname of the machine whose services' data you wish to cache")
arg = arg_parser.parse_args()
log = logs.getLogger(log_name=arg.log_file)
logs.init(arg, l=arg.log_level, logfile=arg.log_file)
conf.init(arg)
assert arg.hostname, '--hostname must be provided'
queues = conf.get_all_cached_queues(hostname=arg.hostname)
pool = []
for queue in queues:
log.info('Spawning cache process for %s' % queue)
p = multiprocessing.Process(name=queue, target=cache_queue, args=(queue,))
p.start()
pool.append(p)
try:
for process in pool:
process.join()
log.warn('%s caching has stopped' % process.name)
except KeyboardInterrupt:
log.warn('Keyboard interrupt in main')
def main():
ap = args.get_parser()
ap.add_argument('--test', action="store_true", help="Test Flag, if contain this argument, it means a test case")
arg = ap.parse_args()
assert arg.sub, 'Need a queue to subscribe to'
assert arg.pub, 'Need a queue to publish to'
logs.init(arg)
queue.init(arg)
test_flag = arg.test
conn = boto.connect_sdb()
with queue.open(arg.sub, 'r') as inq:
for m in inq:
try:
durationProcess(conn, m, arg.pub, test_flag)
except KeyboardInterrupt:
log.info('GOT SIGINT, exiting!')
break
except EmbersException as e:
log.exception(e.value)
except:
log.exception("Unexpected exception in process")
def init():
global con
global cur
con = common.getDBConnection()
cur = con.cursor()
logs.init()
def main():
ap = args.get_parser()
ap.add_argument('-i', '--inputFolder', type=str,
help='inputFolder contaning twitter files',
default='/hdd/tweets/2012/may')
ap.add_argument('-s', '--scoresFolder', type=str,
help='Folder contaning scoreCards',
default='../data/scores/MX/')
ap.add_argument('-cf', '--configFile', type=str,
help='election configuration file',
default='../configFiles/electionConfig_MX')
ap.add_argument('-d1', '--fromDate', type=str,
help='fromDate')
ap.add_argument('-d2', '--toDate', type=str,
help='toDate')
ap.add_argument('-f1', '--flag1', help="countOrPredict",
type=str, default='2')
ap.add_argument('-r', '--regression', help="regressionType",
type=str, default='LASSO')
ap.add_argument('-f2', '--flag2', help="flag to push surrogates and warning to S3",
type=str, default='0')
arg = ap.parse_args()
logs.init(arg)
try:
elections = Elections(arg.inputFolder, arg.scoresFolder,
arg.configFile, arg.fromDate, arg.toDate)
log.info("Election class initialized")
except Exception as e:
log.exception("exception during intialization: %s. Quitting!!", e)
try:
if (arg.flag1 == '1' or arg.flag1 == '3'):
elections.collectMentions()
except Exception as e:
log.exception("error while tracking tweets")
try:
if (arg.flag1 == '2' or arg.flag1 == '3'):
winner, winningScore, runnerUp, runnerUpScore, finalScore = elections.getWinner(arg.fromDate, arg.toDate, arg.regression)
print "------------Regression Results-----------"
print finalScore
print winner + "====>" + str(winningScore)
print "-----------------------------------------"
except Exception as e:
log.exception("error while calculating winner:%s", e)
try:
elections.createSurrogate(winner, winningScore, runnerUp, runnerUpScore, arg.flag2)
except Exception as e:
log.exception("error during creating warnings")
try:
if (arg.flag2 == '1'):
elections.storeStatistics(arg.fromDate, arg.toDate)
except Exception as e:
log.exception("error in storing statistics:%s", e)
log.info("ALL Operations Complete")
def main():
# Initialize arguments
argparser = args.get_parser()
argparser.add_argument('--local_port', help='Local port to connect to java server', required=True)
arg = argparser.parse_args()
localPort = int(arg.local_port)
# Initialize log
logs.init(arg)
global log
# Initialize the queue with arguments and connect to the specified feed
log.info("Opening and connecting to queue %s", arg.sub)
queue.init(arg)
reader = queue.open(arg.sub, 'sub', ssh_key=arg.ssh_key, ssh_conn=arg.tunnel)
# Initialize the writer to publish to a queue
log.info("Publishing to queue %s", arg.pub)
writer = queue.open(arg.pub, 'pub', ssh_key=arg.ssh_key, ssh_conn=arg.tunnel)
count = 0
# Connect to Java server
while True:
for feedmsg in reader:
try:
while True:
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(("localhost", localPort))
break
except:
log.info("Unable to connect to local server")
log.debug("Connected to java server on port %d" % localPort)
socketLines = sock.makefile()
# Clean the message to fix irregularities
feedmsg = message.clean(feedmsg)
log.debug("Read message %d. Sending to java" % count)
# Write message to socket stream
sock.sendall(json.dumps(feedmsg))
sock.sendall('\n')
# Receive result from socket stream
result = socketLines.readline()
writer.write(json.dumps(result))
count += 1
sock.close()
except KeyboardInterrupt:
sys.exit(1)
else:
log.info("Server was disconnected.")
def main():
#initiate parameters
global TREND_RANGE
"Initiate the TimeZone Setting"
arg = parse_args()
conn = boto.connect_sdb()
operate_date = arg.operate_date
start_date = arg.start_date
end_date = arg.end_date
port = arg.pub
assert port, "Need a queue to publish to"
logs.init(arg)
queue.init(arg)
t_domain = get_domain(conn, 't_enriched_bloomberg_prices')
#trend_file = args.trend_file
# "Load the trend changeType range file"
trendObject = None
trendObject = json.load(sys.stdin)
# "Get the latest version of Trend Ranage"
trend_versionNum = max([int(v) for v in trendObject.keys()])
# "To avoid changing the initiate values, we first transfer the json obj to string ,then load it to create a news object"
TREND_RANGE = json.loads(json.dumps(trendObject[str(trend_versionNum)]))
# "If input a date range, then we will handle all the data query from those days"
if start_date is None:
#get raw price list
raw_price_list = []
rs = get_raw_data(conn, operate_date)
for r in rs:
raw_price_list.append(r)
for raw_data in raw_price_list:
process(t_domain, port, raw_data)
else:
t_format = "%Y-%m-%d"
s_date = datetime.strptime(start_date, t_format)
e_date = datetime.strptime(end_date, t_format)
while s_date <= e_date:
raw_price_list = []
rs = get_raw_data(conn, datetime.strftime(s_date, t_format))
for r in rs:
raw_price_list.append(r)
for raw_data in raw_price_list:
process(t_domain, port, raw_data)
s_date = s_date + timedelta(days=1)
# "sleep 5 s to wait simpleDB to commit"
time.sleep(5)
#"Write back the trendFile"
new_version_num = trend_versionNum + 1
trendObject[str(new_version_num)] = TREND_RANGE
json.dump(trendObject, sys.stdout)
def main():
logs.init(l=logs.DEBUG)
# TODO translate non-English characters
# print(get_tweets(keywords='esta'))
# print(aggregate_tweets_over_time())
# print(aggregate_tweets_by_country())
# print(aggregate_tweets_by_country(lemmas='bill'))
# print(aggregate_tweets_by_country(country='brazil', lemmas='bill'))
# print(aggregate_tweets_by_country(country='brazil', lemmas='bill', entities=''))
print(get_abbreviated_tweets_for_dqe('brazil', 'datasift-keyword', start_date='2015-03-27', end_date='2015-03-27'))
def main():
'''
Reads the from the queue, retrieves the content
from the source website and publishes the content to a new queue.
'''
ap = args.get_parser()
ap.add_argument(
'--cat',
action="store_true",
help='Read input from standard in and write to standard out.')
arg = ap.parse_args()
logs.init(arg)
geo_mena = GeoMena()
geo_lac = Geo(geo_region=GEO_REGION.lac)
try:
if arg.cat:
log.debug('Reading from stdin and writing to stdout.')
ins = sys.stdin
outs = sys.stdout
for entry in ins:
entry = entry.decode(encoding='utf-8')
try:
tweet = json.loads(entry.strip())
geo_annotate(tweet, geo_mena, geo_lac)
if tweet is not None:
outs.write(
json.dumps(tweet,
ensure_ascii=False).encode("utf-8"))
outs.write('\n')
outs.flush()
except Exception:
log.exception('Failed to process message "%s".', (entry, ))
else:
queue.init(arg)
with queue.open(arg.sub, 'r') as inq:
with queue.open(arg.pub, 'w', capture=True) as outq:
for tweet in inq:
try:
content = geo_annotate(tweet, geo_mena, geo_lac)
if content is not None:
outq.write(content)
except KeyboardInterrupt:
log.info("Got SIGINT, exiting.")
break
except Exception:
log.exception('Failed to process message "%s".',
(tweet, ))
return 0
except Exception as e:
log.exception("Unknown error in main function-{}".format(str(e)))
return 1
def main():
'''
Reads the from the queue, retrieves the content
from the source website and publishes the content to a new queue.
'''
ap = args.get_parser()
ap.add_argument('--cat', action="store_true",
help='Read input from standard in and write to standard out.')
ap.add_argument('--region', metavar='REGION', type=str, default=None,
help='Specify region to filter by')
arg = ap.parse_args()
logs.init(arg)
filter_region = arg.region
geoc = GeoCountry()
try:
if arg.cat:
log.debug('Reading from stdin and writing to stdout.')
ins = sys.stdin
outs = codecs.getwriter('utf-8')(sys.stdout)
for entry in ins:
entry = entry.decode(encoding='utf-8')
try:
tweet = json.loads(entry.strip())
tweet = annotate(tweet, geoc, filter_region)
if tweet is not None:
outs.write(json.dumps(tweet, ensure_ascii=False))
outs.write('\n')
outs.flush()
except Exception:
log.exception('Failed to process message "%s".', entry)
else:
queue.init(arg)
iqueue.init(arg)
qname = "{}-geoCountry-{}".format(os.environ["CLUSTERNAME"], filter_region)
with iqueue.open(arg.sub, 'r', qname=qname) as inq:
with queue.open(arg.pub, 'w') as outq: # , capture=True) as outq:
for tweet in inq:
try:
content = annotate(tweet, geoc, filter_region)
if content is not None:
outq.write(content)
except KeyboardInterrupt:
log.info("Got SIGINT, exiting.")
break
except Exception:
log.exception('Failed to process message "%s".', tweet)
return 0
except Exception as e:
log.exception("Unknown error in main function-{0!s}.".format(e))
return 1
def main():
ap = args.get_parser()
ap.add_argument('--replay', action="store_true", help="Test Flag, if contain this argument, it means a test case")
#if the rule file is not indicated in argument, it need to be load from sys.stdin
ap.add_argument('--rulefile', type=str, help="The rule file for duration analysis model")
arg = ap.parse_args()
if not arg.replay:
assert arg.sub, 'Need a queue to subscribe to'
assert arg.pub, 'Need a queue to publish to'
logs.init(arg)
queue.init(arg)
test_flag = arg.replay
if arg.rulefile:
rule = eval(open(arg.rulefile).read())
else:
#load the rules from sys.stdin
rule = eval(sys.stdin.read())
conn = boto.connect_sdb()
if not arg.replay:
with queue.open(arg.sub, 'r') as inq:
for m in inq:
try:
replayIO = StringIO.StringIO()
durationProcess(rule, conn, m, arg.pub, test_flag, replayIO)
except KeyboardInterrupt:
log.info('GOT SIGINT, exiting!')
break
except EmbersException as e:
log.exception(e.value)
except:
log.exception("Unexpected exception in process")
else:
#replay model take enriched file as input
enrich_messages = sys.stdin.readlines()
for m in enrich_messages:
m = json.loads(m.strip())
try:
replayIO = StringIO.StringIO()
durationProcess(rule, conn, m, arg.pub, test_flag, replayIO)
except KeyboardInterrupt:
log.info('GOT SIGINT, exiting!')
break
except EmbersException as e:
log.exception(e.value)
except:
log.exception("Unexpected exception in process")
def main():
ap = args.get_parser()
ap.add_argument('--o', type=str, help="the output dir to store news")
arg = ap.parse_args()
assert arg.o, 'Need a dir to store news'
logs.init(arg)
locale.setlocale(locale.LC_TIME, 'es_ES.utf-8')
seen_it = shelve.open('elfinance_seen_it.db')
cas = ['finanzas']
for ca in cas:
get_category_news(ca, seen_it, arg.o)
def main():
'''
Reads the from the queue, retrieves the content
from the source website and publishes the content to a new queue.
'''
ap = args.get_parser()
ap.add_argument('--cat', action="store_true",
help='Read input from standard in and write to standard out.')
arg = ap.parse_args()
logs.init(arg)
geo_mena = GeoMena()
geo_lac = Geo(geo_region=GEO_REGION.lac)
try:
if arg.cat:
log.debug('Reading from stdin and writing to stdout.')
ins = sys.stdin
outs = sys.stdout
for entry in ins:
entry = entry.decode(encoding='utf-8')
try:
tweet = json.loads(entry.strip())
geo_annotate(tweet, geo_mena, geo_lac)
if tweet is not None:
outs.write(json.dumps(tweet, ensure_ascii=False).encode("utf-8"))
outs.write('\n')
outs.flush()
except Exception:
log.exception('Failed to process message "%s".', (entry,))
else:
queue.init(arg)
with queue.open(arg.sub, 'r') as inq:
with queue.open(arg.pub, 'w', capture=True) as outq:
for tweet in inq:
try:
content = geo_annotate(tweet, geo_mena, geo_lac)
if content is not None:
outq.write(content)
except KeyboardInterrupt:
log.info("Got SIGINT, exiting.")
break
except Exception:
log.exception('Failed to process message "%s".', (tweet,))
return 0
except Exception as e:
log.exception("Unknown error in main function-{}".format(str(e)))
return 1
def main():
logs.init(l=logs.DEBUG)
# TODO translate non-English characters
# print(get_tweets(keywords='esta'))
# print(aggregate_tweets_over_time())
# print(aggregate_tweets_by_country())
# print(aggregate_tweets_by_country(lemmas='bill'))
# print(aggregate_tweets_by_country(country='brazil', lemmas='bill'))
# print(aggregate_tweets_by_country(country='brazil', lemmas='bill', entities=''))
print(
get_abbreviated_tweets_for_dqe('brazil',
'datasift-keyword',
start_date='2015-03-27',
end_date='2015-03-27'))
def main():
ap = args.get_parser()
ap.add_argument('--f', type=str, help='the newes file')
arg = ap.parse_args()
assert arg.f, 'Need a file to ingest'
assert arg.pub, 'Need a queue to publish'
logs.init(arg)
queue.init(arg)
with queue.open(arg.pub, 'w') as q_w, open(arg.f, 'r') as f_r:
for line in f_r:
news = json.loads(line)
q_w.write(news)
def main():
"""
Utility for warnings stored in Elasticsearch
--log_file : Path to write the log file to
--log_level : Logging level
"""
from etool import args
global log
arg_parser = args.get_parser()
arg = arg_parser.parse_args()
log = logs.getLogger(log_name=arg.log_file)
logs.init(arg, l=arg.log_level, logfile=arg.log_file)
print(query(max_results=30))
def initiate():
global newsAlreadyDownload
global companyListDir
global port
global newsAlreadDownloadFilePath
global companyList
global dailyNewsOutPath
args = parse_args()
logs.init()
newsAlreadDownloadFilePath = args.f_downloaded
companyListDir = args.f_company_list
port = args.port
dailyNewsOutPath = args.f_out
newsAlreadyDownload = json.load(open(newsAlreadDownloadFilePath))
companyList = json.load(open(companyListDir))
def main():
ap = args.get_parser()
ap.add_argument('--out', help="the output file of warnings")
arg = ap.parse_args()
assert arg.sub, 'Need a queue to subcribe!'
assert arg.out, 'Need a file to store warnings!'
logs.init(arg)
queue.init(arg)
out_file = arg.out
with queue.open(arg.sub, 'r') as q_r:
for m in q_r:
with open(out_file, "a") as out_w:
if not check_ifexist(m):
out_w.write(json.dumps(m) + "\n")
else:
print "Duplicated Warnings"
def main():
ap = args.get_parser()
ap.add_argument('--out', help="the output file of warnings")
arg = ap.parse_args()
assert arg.sub, 'Need a queue to subcribe!'
assert arg.out, 'Need a file to store warnings!'
logs.init(arg)
queue.init(arg)
out_file = arg.out
with queue.open(arg.sub, 'r') as q_r:
for m in q_r:
with open(out_file, "a") as out_w:
if not check_ifexist(m):
out_w.write(json.dumps(m) + "\n")
else:
print "Duplicated Warnings"
def main():
ap = args.get_parser()
ap.add_argument('-i', '--input', default='sys.stdin', type=str, help='Path to the input file.'
'Default is sys.stdin')
ap.add_argument('-o', '--out', default='sys.stdout', type=str, help='Path to the output file.'
'Default is sys.stdout')
ap.add_argument('searchPhrase', default='config/phrases.txt', type=str, help='Path to '
'the Phrase File if "-f" flag is specified, else the input string is considered'
'to be the phrase.')
ap.add_argument('-f', '--file', action='store_true', default=False, help='If given, then the '
'the searchPhrase argument is interpreted as path to a file')
global logger
logger = logs.getLogger("%s-%s.log" % (__processor__, str(datetime.now())))
arg = ap.parse_args()
logs.init(args)
inputFile = None
outFile = None
phraseFile = None
if arg.input == 'sys.stdin':
reader = codecs.getreader('utf-8')(sys.stdin)
else:
inputFile = open(arg.input, "r")
reader = codecs.getreader('utf-8')(inputFile)
if arg.out == 'sys.stdout':
writer = codecs.getwriter('utf-8')(sys.stdout)
else:
outFile = codecs.open(arg.out, "w", encoding="utf-8")
writer = codecs.getwriter('utf-8')(outFile)
if arg.file:
phraseFile = codecs.open(arg.searchPhrase, encoding='utf-8')
generatePhraseList(phraseFile.readlines())
else:
generatePhraseList([arg.searchPhrase])
phraseSearch(reader, writer)
#close all files
if inputFile:
inputFile.close()
if outFile:
outFile.close()
if phraseFile:
phraseFile.close()
def main():
"""
Utility to cache messages from all queues from the --hostname provided with 'cache: true' option set in embers.conf
--hostname : Cache all active queues on this host
--log_file : Path to write the log file to
--log_level : Logging level
"""
from etool import args
global log
arg_parser = args.get_parser()
arg_parser.add_argument(
'--hostname',
metavar='HOSTNAME',
type=str,
default=environ.get('HOSTNAME', None),
help=
"The hostname of the machine whose services' data you wish to cache")
arg = arg_parser.parse_args()
log = logs.getLogger(log_name=arg.log_file)
logs.init(arg, l=arg.log_level, logfile=arg.log_file)
conf.init(arg)
assert arg.hostname, '--hostname must be provided'
queues = conf.get_all_cached_queues(hostname=arg.hostname)
pool = []
for queue in queues:
log.info('Spawning cache process for %s' % queue)
p = multiprocessing.Process(name=queue,
target=cache_queue,
args=(queue, ))
p.start()
pool.append(p)
try:
for process in pool:
process.join()
log.warn('%s caching has stopped' % process.name)
except KeyboardInterrupt:
log.warn('Keyboard interrupt in main')
def main():
"""
Utility to set up a mapping for an EMBERS queue in Elasticsearch
-q | --queue : Queue name to set up the mapping for. Settings are read from embers.conf
--log_file : Path to write the log file to
--log_level : Logging level
"""
from etool import args
global log
arg_parser = args.get_parser()
arg_parser.add_argument('-q', '--queue', help='Queue name to map into Elasticsearch')
arg = arg_parser.parse_args()
assert arg.queue, '--queue must be provided'
log = logs.getLogger(log_name=arg.log_file)
logs.init(arg, l=arg.log_level, logfile=arg.log_file)
add_type(index_name=general.get_index_name(), type_name=arg.queue)
def main():
ap = args.get_parser()
default_day = datetime.strftime(datetime.now(), "%Y-%m-%d")
ap.add_argument("--d", type=str, default=default_day, help="The day to ingest, Format: dd/mm/yyyy")
ap.add_argument("--domain", default="bloomberg_prices", help="The simpleDB table to store raw data")
arg = ap.parse_args()
assert arg.pub, "Need a queue to publish"
logs.init(arg)
queue.init(arg)
with queue.open(arg.pub, "w") as out_q:
for stock in STOCK_CON:
if stock == "COLCAP":
scrape_f = scrape_colcap_url
if stock == "CHILE65":
scrape_f = scrape_chile65_url
msg = ingest_price(arg, stock, scrape_f)
if msg is not None:
out_q.write(msg)
store(arg, msg)
def main():
ap = args.get_parser()
ap.add_argument('-c', '--conf', metavar='CONF', type=str, nargs='?',
default=os.path.join(os.path.dirname(__file__), 'bloomberg_news_ingest.conf'),
help='The location of the configuration file.')
arg = ap.parse_args()
assert arg.pub, "--pub required. Need a queue to publish on"
logs.init(arg)
conf = get_conf(arg.conf)
seen_it = shelve.open("bloomberg_news_seen_it.db")
try:
with queue.open(arg.pub, 'w', capture=True) as outq:
for (index, companies) in conf.items():
for company in companies:
articles = get_stock_news(index, company, seen_it)
for a in articles:
outq.write(a)
except KeyboardInterrupt:
log.info('GOT SIGINT, exiting')
def main():
ap = args.get_parser()
ap.add_argument('--level', type=str, default="0.6", help='The threhold')
ap.add_argument('--svm', action='store_true')
ap.add_argument('--zmq', action='store_true')
ap.add_argument('--surr', type=str, help="surrogate file")
ap.add_argument('--warn', type=str, help="warning file")
arg = ap.parse_args()
logs.init(arg)
queue.init(arg)
assert arg.pub, "Please input a queue to publish warning"
if arg.zmq:
assert arg.sub, "Please input a queue to sub surrogate message"
conn = boto.connect_sdb()
t_domain = get_domain(conn, "s_holiday")
if arg.zmq:
with queue.open(arg.sub, 'r') as inq:
for m in inq:
try:
if arg.svm:
svm_warning(t_domain, m, arg.pub)
else:
warning_center(t_domain, m, arg.pub, float(arg.level))
except KeyboardInterrupt:
log.info('GOT SIGINIT, exiting!')
break
except:
log.exception("Exception in Process:%s" %
sys.exc_info()[0])
else:
with open(arg.warn, "w") as w, open(arg.surr) as r:
if arg.svm:
for m in r:
m = json.loads(m)
warning = svm_warning(t_domain, m, arg.pub)
w.write(json.dumps(warning) + "\n")
def main():
ap = args.get_parser()
ap.add_argument('--level', type=str, default="0.6",
help='The threhold')
ap.add_argument('--svm', action='store_true')
ap.add_argument('--zmq', action='store_true')
ap.add_argument('--surr', type=str, help="surrogate file")
ap.add_argument('--warn', type=str, help="warning file")
arg = ap.parse_args()
logs.init(arg)
queue.init(arg)
assert arg.pub, "Please input a queue to publish warning"
if arg.zmq:
assert arg.sub, "Please input a queue to sub surrogate message"
conn = boto.connect_sdb()
t_domain = get_domain(conn, "s_holiday")
if arg.zmq:
with queue.open(arg.sub, 'r') as inq:
for m in inq:
try:
if arg.svm:
svm_warning(t_domain, m, arg.pub)
else:
warning_center(t_domain, m, arg.pub, float(arg.level))
except KeyboardInterrupt:
log.info('GOT SIGINIT, exiting!')
break
except:
log.exception("Exception in Process:%s" % sys.exc_info()[0])
else:
with open(arg.warn, "w") as w, open(arg.surr) as r:
if arg.svm:
for m in r:
m = json.loads(m)
warning = svm_warning(t_domain, m, arg.pub)
w.write(json.dumps(warning) + "\n")
def main():
ap = args.get_parser()
ap.add_argument('--r_file', type=str, help="The rule file")
ap.add_argument('--o', type=str, help="The output file")
arg = ap.parse_args()
assert arg.r_file, 'Need a rule file'
assert arg.sub, 'Need a queue to subscribe'
assert arg.o, 'Need a file to output'
logs.init(arg)
queue.init(arg)
u_pattern = re.compile("http://(www\.){0,1}[^/]*/[a-z0-9/.\-]*(econ)[a-z0-9\.\-]*", flags=re.I)
c_rule = create_label_rule(arg.r_file)
g_rule = create_gold_lable(arg.r_file)
c_pattern = re.compile(c_rule, flags=re.I)
with queue.open(arg.sub, 'r') as q_r, codecs.open(arg.o, 'a') as f_a:
for news in q_r:
f_news = process(news, u_pattern, c_pattern, g_rule)
if f_news is not None:
f_a.write(json.dumps(f_news) + "\n")
print f_news['date'], f_news['title'], "|", f_news['o_country'], "|", f_news["p_country"]
def main():
logs.init(l=logs.DEBUG)
print(get_data(sources=['Warnings']))
def __init__(self,cfgPath):
common.init(cfgPath)
logs.init()
def execute(arg):
logs.init(arg)
fromDate = datetime.strptime(arg.fromDate, "%d %b %Y")
toDate = datetime.strptime(arg.toDate, "%d %b %Y")
tweetFolder = arg.tweetFolder
country = arg.country
hashTagCounts = {}
uids = {}
# loading twitter handles from a file
with open(arg.seedFile, 'r') as _file:
for line in _file:
handle, candidate = line.strip().split(',')
if candidate not in uids:
uids[candidate] = []
hashTagCounts[candidate] = {}
uids[candidate].append(handle.lower())
else:
uids[candidate].append(handle.lower())
# for geolocation
geo = Geo()
for _file in sorted(os.listdir(tweetFolder)):
fileDate = datetime.strptime(_file[17:27], '%Y-%m-%d')
if (fileDate >= fromDate and fileDate < toDate):
log.info("processing file %s" % (_file))
try:
with open(tweetFolder + "/" + _file, "r") as FILE:
for line in FILE:
try:
jsonTweet = json.loads(line.strip())
dateStr = jsonTweet['interaction']['created_at'][5:16]
tweetDate = datetime.strptime(dateStr, '%d %b %Y')
geoList = geo.geo_normalize(jsonTweet)
city, ctry, state = geoList[:3]
if ctry and (ctry.lower() == country) and (tweetDate >= fromDate) and (tweetDate <= toDate):
userId, realName = None, None
if 'twiiter' in jsonTweet:
if 'user' in jsonTweet['twitter']:
if 'screen_name' in jsonTweet['twitter']['user']:
userId = jsonTweet['twitter']['user']['screen_name'].lower()
if 'name' in jsonTweet['twitter']['user']:
realName = jsonTweet['twitter']['user']['name'].lower()
if userId is None and realName is None:
continue
log.debug('userId or realName is not None')
candidate = getCandidate(userId, realName, uids)
if candidate is not None:
log.debug('found candidate--> ' + candidate)
# prereProcess the tweet
text = jsonTweet["interaction"]["content"]
text = re.sub(URL_REGEX, ' ', text) # remove urls
text = re.sub('[^A-Za-z_@#0-9]', ' ', normalize_str(text, lower=True)) # allow only alphaNumerics and twitter tags
text = re.sub(' +', ' ', text) # remove multiple spaces
hashTags = extract_hash_tags(text)
hashTags = [hashTag for hashTag in hashTags if len(hashTag) > 3]
for hashTag in hashTags:
if hashTag.startswith('#'):
hashTag = hashTag[1:]
if hashTag in hashTagCounts[candidate]:
hashTagCounts[candidate][hashTag] += 1
else:
hashTagCounts[candidate][hashTag] = 1
except Exception, e:
log.exception('error processing tweet %s' % e)
except Exception, f:
log.exception('error processing file %s' % f)
else:
log.debug('skipping file %s ' % _file)
def main():
'''
Reads the from the queue, retrieves the content
from the source website and publishes the content to a new queue.
'''
ap = args.get_parser()
ap.add_argument(
'--cat',
action="store_true",
help='Read input from standard in and write to standard out.')
ap.add_argument('--region',
metavar='REGION',
type=str,
default=None,
help='Specify region to filter by')
arg = ap.parse_args()
logs.init(arg)
filter_region = arg.region
geoc = GeoCountry()
try:
if arg.cat:
log.debug('Reading from stdin and writing to stdout.')
ins = sys.stdin
outs = codecs.getwriter('utf-8')(sys.stdout)
for entry in ins:
entry = entry.decode(encoding='utf-8')
try:
tweet = json.loads(entry.strip())
tweet = annotate(tweet, geoc, filter_region)
if tweet is not None:
outs.write(json.dumps(tweet, ensure_ascii=False))
outs.write('\n')
outs.flush()
except Exception:
log.exception('Failed to process message "%s".', entry)
else:
queue.init(arg)
iqueue.init(arg)
qname = "{}-geoCountry-{}".format(os.environ["CLUSTERNAME"],
filter_region)
with iqueue.open(arg.sub, 'r', qname=qname) as inq:
with queue.open(arg.pub,
'w') as outq: # , capture=True) as outq:
for tweet in inq:
try:
content = annotate(tweet, geoc, filter_region)
if content is not None:
outq.write(content)
except KeyboardInterrupt:
log.info("Got SIGINT, exiting.")
break
except Exception:
log.exception('Failed to process message "%s".',
tweet)
return 0
except Exception as e:
log.exception("Unknown error in main function-{0!s}.".format(e))
return 1
def main():
logs.init(l=logs.DEBUG)
print(get_data(sources=['Warnings']))
def main():
"""
Utility to cache messages from a queue into Elasticsearch
-q | --queue : Read from <queue> and write the messages to Elasticsearch. Settings are read from embers.conf
--log_file : Path to write the log file to
--log_level : Logging level
"""
from etool import args
global log
arg_parser = args.get_parser()
arg_parser.add_argument('-q',
'--queue',
help='Queue name to index into Elasticsearch')
arg_parser.add_argument(
'-s',
'--s3fromq',
action='store_true',
help='ingest from S3 prefix derived from queue name')
arg_parser.add_argument('-p', '--prefix', help='Ingest from prefix')
#arg_parser.add_argument('-t', '--typename', default='noqueue', help='Type for prefix ingest')
arg_parser.add_argument('-t', '--typename', help='Type for prefix ingest')
arg_parser.add_argument(
'-l',
'--tmpcopy',
default='/home/embers/data/tmpcopy',
help='Name of local copy of S3 file (same for all S3 files)')
arg_parser.add_argument('-c',
'--chunk',
type=int,
default=100,
help='Chunk size for S3 ingest')
arg_parser.add_argument('-i',
'--clustername',
help='Clustername to determine index name')
arg_parser.add_argument(
'-w',
'--withbase',
action="store_true",
help="Add basename to prefix when looking for type.")
arg_parser.add_argument('--startdate',
help='start date in format like 2015-01-02')
arg_parser.add_argument('--enddate',
help='end date in format like 2015-01-02')
arg = arg_parser.parse_args()
#assert (arg.queue or (arg.prefix and arg.typename)), 'Either --queue (with optional --s3fromq/--typename) or --prefix with --typename must be provided'
assert (
arg.queue or arg.prefix
), 'Either --queue (with optional --s3fromq/--typename) or --prefix must be provided'
log = logs.getLogger(log_name=arg.log_file)
logs.init(arg, l=arg.log_level, logfile=arg.log_file)
index_name = general.get_index_name(arg.clustername)
queue.init()
if arg.prefix or (arg.queue and arg.s3fromq):
if arg.prefix:
prefix = arg.prefix
# get queue name or its substitute for S3 objects from prefix
if arg.typename:
type_name = arg.typename
else:
type_name = queue.conf.get_prefixpair(
prefix=prefix, includeS3=True, withBasename=arg.withbase)
if not type_name:
log.error("Could not get type from prefix %s" % prefix)
return 1
log.warning("type_name=%s from prefix=%s" %
(type_name, prefix))
else:
type_name = arg.queue
prefix, include = queue.conf.get_prefix_for_queue(
type_name, withBasename=False)
if not prefix:
log.error("Could not get S3 prefix for queue %s" % type_name)
return 1
if not general.get_es_connection().indices.exists_type(
index=index_name, doc_type=type_name):
# Create mapping if the queue has not been stored in Elasticsearch yet
index_setup.add_type(index_name=index_name, type_name=type_name)
conn_s3 = boto.connect_s3(aws_access_key_id=arg.aws_key,
aws_secret_access_key=arg.aws_secret)
bucket = conn_s3.get_bucket(
arg.bucket) # connect to S3, get bucket ptr for arg.bucket
attach_to_s3(index_name,
s3prefix=prefix,
bucket=bucket,
type_name=type_name,
tmpcopy=arg.tmpcopy,
chunk_size=arg.chunk,
startdate=arg.startdate,
enddate=arg.enddate)
else:
if arg.typename:
type_name = arg.typename
else:
type_name = arg.queue
if not general.get_es_connection().indices.exists_type(
index=index_name, doc_type=type_name):
# Create mapping if the queue has not been stored in Elasticsearch yet
index_setup.add_type(index_name=index_name, type_name=type_name)
attach_to_queue(index_name=index_name,
queue_name=arg.queue,
type_name=type_name)
def execute(arg):
logs.init(arg)
log.info("*************************************")
log.info("PSL 4 Elections pipeline initializing")
log.info("tweet folder------> " + arg.tweetFolder)
log.info("dataFolder--------> " + str(arg.dataFolder))
log.info("fromDate----------> " + arg.fromDate)
log.info("toDate------------> " + arg.toDate)
log.info("window------------> " + str(arg.window) + " day(s)")
log.info("userThreshold-----> " + str(arg.userThreshold))
log.info("wordThreshold-----> " + str(arg.wordThreshold))
log.info("makePredictions---> " + arg.predictionFlag)
log.info("*************************************")
fromDate = datetime.strptime(arg.fromDate, "%d %b %Y")
toDate = datetime.strptime(arg.toDate, "%d %b %Y")
currentDate = fromDate
iterCount = 1
membership = {}
likes = {}
hates = {}
if arg.predictionFlag == '1':
makePredictions(arg.dataFolder + '/' + arg.country, fromDate, toDate, arg.userThreshold, arg.configFile, arg.regressionFlag)
sys.exit()
while(currentDate <= toDate):
log.info("iterCount--------------->" + str(iterCount))
log.info("processing PSL pipeline for %s" % (currentDate.strftime("%d %b %Y")))
log.debug("creating the directory substructure for current date")
inputFolder = arg.dataFolder + '/' + arg.country + '/' + currentDate.strftime("%d%b") + '/inputs'
outputFolder = arg.dataFolder + '/' + arg.country + '/' + currentDate.strftime("%d%b") + '/outputs'
statsFolder = arg.dataFolder + '/' + arg.country + '/' + currentDate.strftime("%d%b") + '/stats'
os.system('mkdir -p ' + inputFolder)
os.system('mkdir -p ' + outputFolder)
os.system('mkdir -p ' + statsFolder)
nextDate = currentDate + timedelta(days=arg.window)
if(nextDate <= toDate):
log.debug("creating the directory substructure for next date")
nextInputFolder = arg.dataFolder + '/' + arg.country + '/' + nextDate.strftime("%d%b") + '/inputs'
nextOutputFolder = arg.dataFolder + '/' + arg.country + '/' + nextDate.strftime("%d%b") + '/outputs'
nextStatsFolder = arg.dataFolder + '/' + arg.country + '/' + nextDate.strftime("%d%b") + '/stats'
os.system('mkdir -p ' + nextInputFolder)
os.system('mkdir -p ' + nextOutputFolder)
os.system('mkdir -p ' + nextStatsFolder)
if iterCount == 1:
keywordList = []
# adding seed words to the list
seedWordList = []
with open(arg.seedFile, 'r') as file:
for line in file:
word, group, weight = line.split(',')
seedWordList.append(word)
keywordList = list(set(keywordList).union(set(seedWordList)))
log.info("copying the seedFile to inputFolder")
os.system('cp ' + arg.seedFile + ' ' + inputFolder + '/seedWords.csv')
preProcess(arg.tweetFolder, inputFolder, keywordList, currentDate, nextDate, arg.country)
log.info("***********preProcess complete******************")
executePSLCode(inputFolder, outputFolder, arg.classPathFile)
log.info("***********PSL code complete********************")
postProcess(outputFolder, nextInputFolder, statsFolder, arg.userThreshold, arg.wordThreshold, membership, likes, hates)
log.info("**********postProcess complete*****************")
log.info("deleting the database used for current iteration")
os.system('rm /home/aravindan/Dropbox/git/ms_thesis/psl/electionLDADB*')
iterCount += 1
currentDate = nextDate
log.info("**********************************************************")
import xlrd
import json
import datetime
import hashlib
import os
import argparse
import sys
from etool import logs
from Util import calculator
import sqlite3 as lite
"""
Initiate the Json data for all the Indices
"""
logs.init()
__processor__ = os.path.basename(__file__.split(".")[0])
log = logs.getLogger(__processor__)
def transcsv2json(xlsFile):
wb = xlrd.open_workbook(xlsFile)
sh = wb.sheet_by_name('Sheet1')
#read the stock index from file
stockIndex = sh.row_values(0,0)[0].split(" ")[0]
stockPrices = []
for rownum in range(2,sh.nrows):
try:
time_tuple = xlrd.xldate_as_tuple(sh.row_values(rownum,0)[0],0)
post_date = datetime.datetime.strftime(datetime.date(time_tuple[0],time_tuple[1],time_tuple[2]),"%Y-%m-%d")
lastPrice = float(sh.row_values(rownum,0)[1])
previousCloseValue = float(sh.row_values(rownum,0)[3])
def main():
ap = args.get_parser()
ap.add_argument('-t', '--tweetFolder', type=str,
help='inputFolder pointing to PSLs output',
default='/hdd/tweets/2012/oct')
ap.add_argument('-df', '--dataFolder', type=str,
help='folder to store intermediate outputs and final outputs',
default='/home/aravindan/Dropbox/git/ms_thesis/data/psl')
ap.add_argument('-ut', '--userThreshold', type=float,
help='probability threshold of user membership',
default=0.60)
ap.add_argument('-wt', '--wordThreshold', type=float,
help='probability threshold for vocab',
default=0.70)
ap.add_argument('-s', '--seedFile', type=str,
help='seed File containing the intial seed vocabulary',
default='/home/aravindan/Dropbox/git/ms_thesis/psl/seedWords/venezuela.csv')
ap.add_argument('-c', '--country', type=str,
help='country to model elections for',
default='venezuela')
ap.add_argument('-w', '--window', type=int,
help='number of days of tweets used to infer',
default=1)
ap.add_argument('-d1', '--fromDate', type=str,
help='date from which to track tweets',
default='01 Oct 2012')
ap.add_argument('-d2', '--toDate', type=str,
help='date to which to track tweets',
default='06 Oct 2012')
ap.add_argument('-cp', '--classPathFile', type=str,
help='file containing class path for PSL execution',
default='/home/aravindan/Dropbox/git/ms_thesis/psl/classPathFile.txt')
arg = ap.parse_args()
logs.init(arg)
log.info("*************************************")
log.info("PSL 4 Elections pipeline initializing")
log.info("tweet folder------> " + arg.tweetFolder)
log.info("dataFolder--------> " + str(arg.dataFolder))
log.info("fromDate----------> " + arg.fromDate)
log.info("toDate------------> " + arg.toDate)
log.info("window------------> " + str(arg.window) + " day(s)")
log.info("userThreshold-----> " + str(arg.userThreshold))
log.info("userThreshold-----> " + str(arg.userThreshold))
log.info("*************************************")
fromDate = datetime.strptime(arg.fromDate, "%d %b %Y")
toDate = datetime.strptime(arg.toDate, "%d %b %Y")
currentDate = fromDate
iterCount = 1
membership = {}
vocab = {}
filesProcessed = []
while(currentDate <= toDate):
log.info("iterCount--------------->" + str(iterCount))
log.info("processing PSL pipeline for %s" % (currentDate.strftime("%d %b %Y")))
log.info("creating the directory substructure for current iteration")
inputFolder = arg.dataFolder + '/' + arg.country + '/iteration' + str(iterCount) + '/inputs'
outputFolder = arg.dataFolder + '/' + arg.country + '/iteration' + str(iterCount) + '/outputs'
statsFolder = arg.dataFolder + '/' + arg.country + '/iteration' + str(iterCount) + '/stats'
os.system('mkdir -p ' + inputFolder)
os.system('mkdir -p ' + outputFolder)
os.system('mkdir -p ' + statsFolder)
log.info("creating the directory substructure for next iteration")
nextInputFolder = arg.dataFolder + '/' + arg.country + '/iteration' + str(iterCount + 1) + '/inputs'
nextOutputFolder = arg.dataFolder + '/' + arg.country + '/iteration' + str(iterCount + 1) + '/outputs'
nextStatsFolder = arg.dataFolder + '/' + arg.country + '/iteration' + str(iterCount + 1) + '/stats'
os.system('mkdir -p ' + nextInputFolder)
os.system('mkdir -p ' + nextOutputFolder)
os.system('mkdir -p ' + nextStatsFolder)
if iterCount == 1:
keywordList = []
# adding seed words to the list
seedWordList = []
with open(arg.seedFile, 'r') as file:
for line in file:
word, group, weight = line.split(',')
seedWordList.append(word)
keywordList = list(set(keywordList).union(set(seedWordList)))
log.info("copying the seedFile to inputFolder")
os.system('cp ' + arg.seedFile + ' ' + inputFolder + '/seedWords.csv')
fileDate, filesProcessed = preProcess(arg.tweetFolder, inputFolder, keywordList, currentDate, toDate, arg.country, list(set(filesProcessed)))
log.info("***********preProcess complete******************")
executePSLCode(inputFolder, outputFolder, arg.classPathFile)
log.info("***********PSL code complete********************")
keywordList = postProcess(outputFolder, nextInputFolder, statsFolder, arg.userThreshold, arg.wordThreshold, membership, vocab, seedWordList, currentDate)
log.info("**********postProcess complete*****************")
if fileDate > currentDate:
currentDate = fileDate
log.info("deleting the database used for current iteration")
os.system('rm /home/aravindan/Dropbox/git/ms_thesis/psl/electionPSLDB*')
iterCount += 1
log.info("**********************************************************")
def main():
ap = args.get_parser()
ap.add_argument(
"-t", "--tweetFolder", type=str, help="inputFolder pointing to PSLs output", default="/hdd/tweets/2012/oct"
)
ap.add_argument(
"-df",
"--dataFolder",
type=str,
help="folder to store intermediate outputs and final outputs",
default="/home/aravindan/Dropbox/git/ms_thesis/data/dqe",
)
ap.add_argument(
"-wt", "--wordThreshold", type=float, help="n-percent of words to propagate to next iteration", default=25
)
ap.add_argument(
"-s",
"--seedFile",
type=str,
help="seed File containing the intial seed vocabulary",
default="/home/aravindan/Dropbox/git/ms_thesis/psl/seedWords/venezuela.csv",
)
ap.add_argument("-d1", "--fromDate", type=str, help="date from which to track tweets", default="01 Oct 2012")
ap.add_argument("-d2", "--toDate", type=str, help="date to which to track tweets", default="06 Oct 2012")
ap.add_argument("-w", "--window", type=int, help="number of days of tweets used to infer", default=1)
ap.add_argument("-c", "--country", type=str, help="country to execute the pipeline for", default="venezuela")
arg = ap.parse_args()
logs.init(arg)
log.info("*************************************")
log.info("PSL 4 Elections pipeline initializing")
log.info("tweet folder------> " + arg.tweetFolder)
log.info("dataFolder--------> " + str(arg.dataFolder))
log.info("fromDate----------> " + arg.fromDate)
log.info("toDate------------> " + arg.toDate)
log.info("window------------> " + str(arg.window) + " day(s)")
log.info("country-----------> " + arg.country)
log.info("wordThreshold-----> " + str(arg.wordThreshold) + "%")
log.info("*************************************")
fromDate = datetime.strptime(arg.fromDate, "%d %b %Y")
toDate = datetime.strptime(arg.toDate, "%d %b %Y")
currentDate = fromDate
iterCount = 1
vocab = {}
while currentDate <= toDate:
log.info("iterCount--------------->" + str(iterCount))
log.info("processing PSL pipeline for %s" % (currentDate.strftime("%d %b %Y")))
log.info("creating the directory substructure for current date")
outputFolder = arg.dataFolder + "/" + arg.country + "/" + currentDate.strftime("%d%b")
os.system("mkdir -p " + outputFolder)
nextDate = currentDate + timedelta(days=arg.window)
if iterCount == 1:
with open(arg.seedFile, "r") as file:
for line in file:
word, group, weight = line.split(",")
if group in vocab:
vocab[group][word] = weight
else:
vocab[group] = {}
vocab[group][word] = weight
counts = trackTweets(arg.tweetFolder, vocab, currentDate, nextDate, arg.country, arg.wordThreshold)
log.info("***********trackTweets complete***************")
# normalize the counts for each group
for group in counts:
max = 0
for word in counts[group]:
if counts[group][word] > max:
max = counts[group][word]
for word in counts[group]:
weight = counts[group][word] / max
vocab[group][word] = weight
# dumping the vocab learnt
for group in vocab:
with open(outputFolder + "/" + group + "_vocab.csv", "w") as f:
sorted_tuples = sorted(vocab[group].iteritems(), key=operator.itemgetter(1), reverse=True)
for (word, weight) in sorted_tuples:
f.write(word + "," + str(weight))
f.write("\n")
currentDate = nextDate
iterCount += 1
log.info("*********************************************")
log.info("************ALL iterations complete********************")
def main():
global CONFIG, VOCABULARY_FILE, WARNING_PORT, SURROGATE_PORT, __version__, KEY_ID, SECRET, T_EASTERN, T_UTC
"Initiate the TimeZone Setting"
T_UTC = pytz.utc
T_EASTERN = pytz.timezone("US/Eastern")
"Get the input arg"
arg = parse_args()
rege_date = arg.rege_date
KEY_ID = arg.aws_key
SECRET = arg.aws_secret
logs.init(arg)
queue.init(arg)
conn = boto.connect_sdb(KEY_ID, SECRET)
#initiate simpleDB domains
surrogateDomain = get_domain(conn, arg.surrogate_domain)
warningDomain = get_domain(conn, arg.warning_domain)
WARNING_PORT = arg.warning_port
SURROGATE_PORT = arg.surrogate_port
"if rege_date is not none, it means to regenerate the past day's prediction"
if not rege_date:
"Normal predict"
predict_date = arg.predict_date
model_cfg = arg.model_cfg
stock_list = None
if arg.stock_list:
stock_list = arg.stock_list
"Get the Latest version of Config Object"
f = open(model_cfg, "r")
configObj = json.load(f)
f.close()
con_versionNum = max([int(v) for v in configObj.keys()])
CONFIG = configObj[str(con_versionNum)]
"Get the Latest version of Trend Range Object"
clusterTrends = json.load(sys.stdin)
trend_versionNum = max([int(v) for v in clusterTrends.keys()])
CONFIG["trendRange"] = {"version": str(trend_versionNum), "range": clusterTrends[str(trend_versionNum)]}
if not stock_list:
stock_list = CONFIG["stocks"]
#"Retrain the model configuration if current day is Saturday"
#weekDay = datetime.strptime(predict_date,"%Y-%m-%d").weekday()
#if weekDay == 5:
# finalClusterProbability, finalClusterMatrix = re_training(surrogateDomain, predict_date, stock_list)
# new_config = json.loads(json.dumps(CONFIG))
# new_config["clusterProbability"] = finalClusterProbability
# new_config["clusterContribution"] = finalClusterMatrix
# "Write back to configure file"
# new_version_num = con_versionNum + 1
# new_config["version"] = new_version_num
# configObj[str(new_version_num)] = new_config
# with open(model_cfg, "w") as out_q:
# out_q.write(json.dumps(configObj))
"Process stock each by each"
for stock in stock_list:
surrogate = process_single_stock(surrogateDomain, predict_date,stock)
if surrogate:
warning = warning_check(warningDomain, surrogate)
else:
"regenerate the old prediction"
model_cfg = arg.model_cfg
stock_list = None
if arg.stock_list:
stock_list = arg.stock_list
"Get the version of Config Object for the indicated prediction"
versionObj = get_predicion_version(warningDomain, rege_date)
configVersionNum = versionObj["configVersion"]
trendVersionNum = versionObj["trendVersion"]
configObj = json.load(open(model_cfg))
if configVersionNum in configObj:
CONFIG = configObj[configVersionNum]
else:
CONFIG = configObj["1"]
"Get the Latest version of Trend Range Object"
clusterTrends = json.load(sys.stdin)
#get the latest version of the warning
tmpVersion = int(trendVersionNum)
while tmpVersion >= 1:
if str(tmpVersion) in clusterTrends:
trendVersionNum = str(tmpVersion)
break
else:
tmpVersion -= 1
CONFIG["trendRange"] = {"version": str(trendVersionNum), "range": clusterTrends[trendVersionNum]}
if not stock_list:
stock_list = CONFIG["stocks"]
"Process stock each by each"
for stock in stock_list:
replayIO = StringIO.StringIO()
surrogate = process_single_stock(surrogateDomain, rege_date, stock, True, replayIO)
if surrogate:
warning = warning_check(warningDomain, surrogate, True, replayIO)
replayInfo = replayIO.getvalue()
weid = getwarningeid(surrogateDomain, rege_date, stock)
with open("./demo/%s.txt" % weid, "w") as win:
win.write(replayInfo)
if conn:
conn.close()
def __init__(self, wg_data=WG_DATA, co_admin_data=CO_ADMIN_DATA,
priority_policy=PRIORITY_POLICY,
debug=False):
"""
"""
self.priority_policy = priority_policy
self.debug = debug
self.__version__ = "{0}-{1}-{2}-{3}-{4}".format(
self.__class__.__name__,
__version__,
hashlib.md5(get_wg_data(wg_data).read()).hexdigest(),
hashlib.md5(get_co_admin_data(co_admin_data).read()).hexdigest(),
hashlib.md5(" ".join(self.priority_policy)).hexdigest())
if self.debug:
try:
logs.init()
except IOError: # , err:
logs.init(logfile=self.__class__.__name__.lower())
self.log = logs.getLogger("{0}-{1}".format(
self.__class__.__name__,
__version__.replace('.', '_')))
# 1. load country and admin1 level geo data
f = get_co_admin_data(co_admin_data)
dialect = csv.Sniffer().sniff(f.read(10240), delimiters="\t")
f.seek(0)
reader = csv.DictReader(f, dialect=dialect, fieldnames=CO_ADMIN_FIELDS)
# NOTE:
# Known conflicts b/w codes of countries and other admins
# co Colombia ('Colombia', 'C\xc3\xb3rdoba')
# cl Chile ('Colombia', 'Caldas')
# ar Argentina ('Colombia', 'Arauca')
# sv El Salvador ('El Salvador', 'San Vicente')
# prep lookup dictionaries
# key__value
# countries
self.co_code = {}
self.co_names = {}
self.co_aliases = {}
self.co_capital_cities = {}
# admin1
self.admin_code = {}
self.admin_name = {}
# assumes countries appear first when reading data from
# lac_co_admin TODO BAD!
for r in reader:
for k in r.keys():
r[k] = r[k].strip()
lat = float_or_none(r['latitude'])
lon = float_or_none(r['longitude'])
code = object_or_none(r['iso_3166_code'])
rid = int_or_none(r["id"])
if r['type'] == 'country':
# country
if code:
self.co_code[code] = r['name']
self.co_names[nstr(r['name'])] = (rid, lat, lon,
code, r['name'])
self.co_capital_cities[nstr(r['capital_city'])] =\
(r['capital_city'], r['name'])
aliases = r['alt_names'].split(',')
self.co_aliases.update({nstr(alias.strip()): r['name']
for alias in aliases})
else:
if self.debug:
self.log.error("Bad data country {0} Code {1}".format(
r['name'], code))
elif r['type'] == 'admin':
# admin
admin, co = r['full_name'].split(',')
admin, co = admin.strip(), co.strip()
if code:
if code not in self.admin_code:
self.admin_code[code] = []
self.admin_code[code].append((co, admin))
co1, a = nstr(co), nstr(admin)
if a not in self.admin_name:
self.admin_name[a] = {}
if co1 not in self.admin_name[a]:
self.admin_name[a][co1] = (rid, lat, lon, code, admin, co)
f.close()
# 2. load (world-gazeteer) city level geo data
f = get_wg_data(wg_data)
dialect = csv.Sniffer().sniff(f.read(10240), delimiters="\t")
f.seek(0)
reader = csv.DictReader(f, dialect=dialect, fieldnames=WG_FIELDS)
self.ci_aliases = {}
# main data store for geocoding
self.data = []
counter = 0
ci_set = set()
for r in reader:
for k in r.keys():
r[k] = r[k].strip()
# get alias names for cities
ci_names = [a.strip() for a in r['alt_names'].split(',')
if len(a.strip()) > 0]
ci_names.extend([a.strip() for a in r['orig_names'].split(',')
if len(a.strip()) > 0])
for ci in ci_names:
k = (nstr(ci), nstr(r['country']))
a1 = nstr(r['admin1'])
if k not in self.ci_aliases:
self.ci_aliases[k] = {a1: set([r['name']])}
elif a1 not in self.ci_aliases[k]:
self.ci_aliases[k][a1] = set([r['name']])
else:
# Cases where different cities for same
# admin-country pair have the same alias
self.ci_aliases[k][a1].add(r['name'])
# add ci name aliases into ci_set
ci_set.add(nstr(ci))
# store only cannonical cities names
self.data.append((counter, (r['name'], r['country'],
r['admin1'],
object_or_none(r['admin2']),
object_or_none(r['admin3']),
int_or_none(r['pop']),
float_or_none(r['latitude']) / 100,
float_or_none(r['longitude']) / 100,
int(r['id']), int(r['padded']))))
counter += 1
self.coordinates = {}
# cases where admin1 and city share the same name
# extended feature/hack #1 to resolve city when
# only country and admin1 are specified
self.same_ci_a1_name = {}
for i, (n, c, a1, a2, a3, p, lat, lon, i_d, pad) in self.data:
nn, nc, na1 = nstr(n), nstr(c), nstr(a1)
self.coordinates[(lat, lon)] = i
if nn == na1 and pad == 0:
self.same_ci_a1_name[(nc, na1)] = n
ci_set.add(nn)
# store (lat, lon)
self.kdtree = KDTree([[i, j] for i, j in self.coordinates.keys()
if i is not None and j is not None])
# build regular expr dicts
co_set = set(self.co_names.keys())
# add country name aliases into co_set
co_set.update(self.co_aliases.keys())
self.co_reg = ManyRE(co_set)
self.ci_reg = ManyRE(ci_set)
# add admin1 name aliases into admin1_set
admin1_set = set(self.admin_name.keys())
# build regular expression stores for co-admin1-ci
self.admin1_reg = ManyRE(admin1_set)
# add stopwords to prevent any 2-letter word in common usage
# to be mis-interpretted as country or admin code
two_letter_stop_words = set(
['BE', 'WE', '\xc3\xa0', 'YO', 'DO', 'YA', 'DE', 'DA', 'HA', 'BY',
'HE', 'AL', 'NI', 'LE', 'NO', 'LO', 'TU', 'TO', 'TI', 'TE', 'EM',
'EL', 'EN', 'IS', 'OS', 'AM', 'IT', 'AO', 'AN', 'AS', 'AT', 'IN',
'EU', 'ES', 'IF', 'ME', 'ON', 'OF', 'LA', 'MI', 'UP', 'SU', 'UM',
'UN', 'SO', 'NA', 'OU', 'MY', 'OR', 'SE', 'US'])
self.co_code_reg = ManyRE([sw for sw in self.co_code.keys()
if sw not in two_letter_stop_words])
self.admin1_code_reg1 = ManyRE(self.admin_code.keys())
self.admin1_code_reg2 = ManyRE([sw for sw in self.admin_code.keys()
if sw not in two_letter_stop_words])
self.bguess = {}
for i, (city, country, admin1, a2, a3, p, la, lo, i_d, pad)\
in self.data:
ci, co, a = nstr(city), nstr(country), nstr(admin1)
# value is list of admin1's that correspond to ci-co key
# ci-co makes dictionary flatter
# choose not to use co-admin1-ci as key to add more flexibility
# for lookups
if ci in self.bguess:
if co in self.bguess[ci]:
if a in self.bguess[ci][co]:
# store original wg-records marked with pad = 0
# to head of the queue
if pad == 0:
self.bguess[ci][co][a].appendleft(i)
else:
self.bguess[ci][co][a].append(i)
else:
self.bguess[ci][co][a] = deque([i])
else:
self.bguess[ci][co] = {a: deque([i])}
else:
self.bguess[ci] = {co: {a: deque([i])}}