def test_range_dates(self):
"Test range_dates function"
date1 = '20160129'
date2 = '20160201'
dates = range_dates([date1,date2])
expect = ['2016/01/29', '2016/01/30', '2016/01/31', '2016/02/01']
self.assertEqual(expect, dates)
date1 = '20151229'
date2 = '20160101'
dates = range_dates([date1,date2])
expect = ['2015/12/29', '2015/12/30', '2015/12/31', '2016/01/01']
self.assertEqual(expect, dates)
date1 = 20160101
date2 = 20160102
dates = range_dates([date1,date2])
expect = ['2016/01/01', '2016/01/02']
self.assertEqual(expect, dates)
date1 = 20160101
date2 = 20160101
dates = range_dates([date1,date2])
expect = ['2016/01/01']
self.assertEqual(expect, dates)
def test_range_dates(self):
"Test range_dates function"
date1 = "20160129"
date2 = "20160201"
dates = range_dates([date1, date2])
expect = ["2016/01/29", "2016/01/30", "2016/01/31", "2016/02/01"]
self.assertEqual(expect, dates)
date1 = "20151229"
date2 = "20160101"
dates = range_dates([date1, date2])
expect = ["2015/12/29", "2015/12/30", "2015/12/31", "2016/01/01"]
self.assertEqual(expect, dates)
date1 = 20160101
date2 = 20160102
dates = range_dates([date1, date2])
expect = ["2016/01/01", "2016/01/02"]
self.assertEqual(expect, dates)
date1 = 20160101
date2 = 20160101
dates = range_dates([date1, date2])
expect = ["2016/01/01"]
self.assertEqual(expect, dates)
def main():
"Main function"
optmgr = OptionParser()
opts = optmgr.parser.parse_args()
time0 = time.time()
if opts.scripts:
scripts()
sys.exit(0)
todate = datetime.datetime.today()
todate = int(todate.strftime("%Y%m%d"))
fromdate = datetime.datetime.today()-datetime.timedelta(days=1)
fromdate = int(fromdate.strftime("%Y%m%d"))
spec = json.load(open(opts.spec)) if opts.spec else {}
timerange = spec.get('spec', {}).get('timerange', [fromdate, todate])
if opts.hdir == HDIR:
hdir = opts.hdir.split()
if len(hdir) == 1:
hdir = hdir[0]
hdirs = []
for tval in range_dates(timerange):
if hdir.find(tval) == -1:
hdirs.append(os.path.join(hdir, tval))
hdir = hdirs
else:
hdir = opts.hdir
results = run(opts.schema, hdir, opts.script, opts.spec, opts.verbose, opts.rout, opts.yarn)
if opts.store:
data = {"results":results,"ts":time.time(),"etime":time.time()-time0}
if opts.wmaid:
data['wmaid'] = opts.wmaid
else:
data['wmaid'] = wmaHash(data)
data['dtype'] = 'job'
pdata = dict(job=data)
postdata(opts.store, pdata, opts.ckey, opts.cert, opts.verbose)
elif opts.amq:
creds = credentials(opts.amq)
host, port = creds['host_and_ports'].split(':')
if creds and StompAMQ:
print("### Send %s docs via StompAMQ" % len(results))
amq = StompAMQ(creds['username'], creds['password'], \
creds['producer'], creds['topic'], [(host, port)])
amq.send(results)
else:
print(results)
def main():
"Main function"
optmgr = OptionParser()
opts = optmgr.parser.parse_args()
time0 = time.time()
if opts.scripts:
scripts()
sys.exit(0)
verbose = opts.verbose
todate = datetime.datetime.today()
todate = int(todate.strftime("%Y%m%d"))
fromdate = datetime.datetime.today() - datetime.timedelta(days=1)
fromdate = int(fromdate.strftime("%Y%m%d"))
spec = {}
try:
if os.path.isfile(opts.spec):
spec = json.load(open(opts.spec))
else:
spec = json.loads(opts.spec)
except Exception as exp:
pass
timerange = spec.get('spec', {}).get('timerange', [fromdate, todate])
if timerange and verbose:
print("### TimeRang: %s" % timerange)
hdir = opts.hdir
if timerange:
pat = re.compile(".*/20[0-9][0-9].*")
if len(hdir.split()) == 1 and not pat.match(hdir):
hdir = hdir.split()[0]
hdirs = []
for tval in range_dates(timerange):
if hdir.find(tval) == -1:
hdirs.append(os.path.join(hdir, tval))
hdir = hdirs
if verbose:
print("### HDIR: %s" % hdir)
results = run(opts.schema, hdir, opts.script, opts.spec, verbose,
opts.rout, opts.yarn)
if opts.store:
data = {
"results": results,
"ts": time.time(),
"etime": time.time() - time0
}
if opts.wmaid:
data['wmaid'] = opts.wmaid
else:
data['wmaid'] = wmaHash(data)
data['dtype'] = 'job'
pdata = dict(job=data)
postdata(opts.store, pdata, opts.ckey, opts.cert, verbose)
elif opts.amq:
creds = credentials(opts.amq)
host, port = creds['host_and_ports'].split(':')
port = int(port)
if creds and StompAMQ:
print("### Send %s docs via StompAMQ" % len(results))
amq = StompAMQ(creds['username'], creds['password'], \
creds['producer'], creds['topic'], \
validation_schema=None, \
host_and_ports=[(host, port)])
data = []
for doc in results:
hid = doc.get("hash", 1)
if '_id' in doc:
del doc['_id'] # delete ObjectID from MongoDB
producer = "wmarchive"
tstamp = int(time.time()) * 1000
notification, _, _ = amq.make_notification(doc,
hid,
producer=producer,
ts=tstamp,
dataSubfield="")
data.append(notification)
results = amq.send(data)
print("### results from AMQ %s" % len(results))
else:
if isinstance(results, list):
print("### number of results %s" % len(results))
for doc in results:
if '_id' in doc:
del doc['_id'] # delete ObjectID from MongoDB
try:
print(json.dumps(doc))
except:
print(doc)
else:
print(results)
def main():
"Main function"
optmgr = OptionParser()
opts = optmgr.parser.parse_args()
time0 = time.time()
if opts.scripts:
scripts()
sys.exit(0)
verbose = opts.verbose
todate = datetime.datetime.today()
todate = int(todate.strftime("%Y%m%d"))
fromdate = datetime.datetime.today() - datetime.timedelta(days=1)
fromdate = int(fromdate.strftime("%Y%m%d"))
spec = {}
try:
if os.path.isfile(opts.spec):
spec = json.load(open(opts.spec))
else:
spec = json.loads(opts.spec)
except Exception as exp:
pass
timerange = spec.get('spec', {}).get('timerange', [fromdate, todate])
if timerange and verbose:
print("### TimeRang: %s" % timerange)
hdir = opts.hdir
if timerange:
pat = re.compile(".*/20[0-9][0-9].*")
if len(hdir.split()) == 1 and not pat.match(hdir):
hdir = hdir.split()[0]
hdirs = []
for tval in range_dates(timerange):
if hdir.find(tval) == -1:
hdfs_file_path = os.path.join(hdir, tval)
# check whether the hdfs path exists
cmd = ['hdfs', 'dfs', '-test', '-d', hdfs_file_path]
ret, out, err = run_cmd(cmd)
if ret == 0:
hdirs.append(hdfs_file_path)
else:
print "Path does not exist:", hdfs_file_path
hdir = hdirs
if verbose:
print("### HDIR: %s" % hdir)
if opts.logfail:
print "Start query"
runActionsHistoryQuery(opts.schema, hdir, verbose, opts.yarn)
print "Finish query"
else:
results = run(opts.schema, hdir, opts.script, opts.spec, verbose,
opts.rout, opts.yarn)
if opts.store:
data = {
"results": results,
"ts": time.time(),
"etime": time.time() - time0
}
if opts.wmaid:
data['wmaid'] = opts.wmaid
else:
data['wmaid'] = wmaHash(data)
data['dtype'] = 'job'
pdata = dict(job=data)
postdata(opts.store, pdata, opts.ckey, opts.cert, verbose)
elif opts.amq:
creds = credentials(opts.amq)
host, port = creds['host_and_ports'].split(':')
port = int(port)
if creds and StompAMQ:
print("### Send %s docs via StompAMQ" % len(results))
amq = StompAMQ(creds['username'], creds['password'], \
creds['producer'], creds['topic'], [(host, port)])
data = []
for doc in results:
hid = doc.get("hash", 1)
if '_id' in doc:
del doc['_id'] # delete ObjectID from MongoDB
data.append(amq.make_notification(doc, hid))
results = amq.send(data)
print("### results from AMQ %s" % len(results))
else:
if isinstance(results, list):
print("### number of results %s" % len(results))
for doc in results:
if '_id' in doc:
del doc['_id'] # delete ObjectID from MongoDB
try:
print(json.dumps(doc))
except:
print(doc)
else:
print(results)
def make_hdfs_path(hdir, trange):
"""
Create an HDFS paths to look at from provided main hdfs dir
and provided time range list.
"""
return ['%s/%s' % (hdir, d) for d in range_dates(trange)]
def main():
"Main function"
optmgr = OptionParser()
opts = optmgr.parser.parse_args()
time0 = time.time()
if opts.scripts:
scripts()
sys.exit(0)
todate = datetime.datetime.today()
todate = int(todate.strftime("%Y%m%d"))
fromdate = datetime.datetime.today() - datetime.timedelta(days=1)
fromdate = int(fromdate.strftime("%Y%m%d"))
spec = {}
try:
if os.path.isfile(opts.spec):
spec = json.load(open(opts.spec))
else:
spec = json.loads(opts.spec)
except Exception as exp:
pass
timerange = spec.get('spec', {}).get('timerange', [fromdate, todate])
if opts.hdir == HDIR:
hdir = opts.hdir.split()
if len(hdir) == 1:
hdir = hdir[0]
hdirs = []
for tval in range_dates(timerange):
if hdir.find(tval) == -1:
hdirs.append(os.path.join(hdir, tval))
hdir = hdirs
else:
hdir = opts.hdir
results = run(opts.schema, hdir, opts.script, opts.spec, opts.verbose,
opts.rout, opts.yarn)
if opts.store:
data = {
"results": results,
"ts": time.time(),
"etime": time.time() - time0
}
if opts.wmaid:
data['wmaid'] = opts.wmaid
else:
data['wmaid'] = wmaHash(data)
data['dtype'] = 'job'
pdata = dict(job=data)
postdata(opts.store, pdata, opts.ckey, opts.cert, opts.verbose)
elif opts.amq:
creds = credentials(opts.amq)
host, port = creds['host_and_ports'].split(':')
port = int(port)
if creds and StompAMQ:
print("### Send %s docs via StompAMQ" % len(results))
amq = StompAMQ(creds['username'], creds['password'], \
creds['producer'], creds['topic'], [(host, port)])
data = []
for doc in results:
hid = doc.get("hash", 1)
if '_id' in doc:
del doc['_id'] # delete ObjectID from MongoDB
data.append(amq.make_notification(doc, hid))
results = amq.send(data)
print("### results from AMQ", len(results))
else:
print("### agg. results", len(results))
def make_hdfs_path(hdir, trange):
"""
Create an HDFS paths to look at from provided main hdfs dir
and provided time range list.
"""
return ['%s/%s' % (hdir, d) for d in range_dates(trange)]
