import boto, time, json, pprint
from datetime import datetime, timedelta
import numpy as np
import fetch.myS3 as myS3
import fetch.mySQS as mySQS
import Simscore.validity_metrics as vm
from fetch.aws import aws_ak, aws_sk
# <codecell>
conn = boto.connect_s3(aws_ak, aws_sk)
bucket = conn.get_bucket('incoming-simscore-org')
mind = mindate=datetime.utcnow()-timedelta(days=6)
print mind, datetime.utcnow()
allfiles = myS3.getFilesBetween(mindate=mind, bucket=bucket, onlyTxtFiles=True)
# <codecell>
userfiles = []
for f in allfiles:
edge = f.split('/')[0][4:]
if f.split('.')[-3] != '109' and edge not in ['0','11','12']:
userfiles.append(f)
ffs = myS3.getTestFiles(userfiles, bucket)
print ffs
# <codecell>
report = {}
minmax = validate.findMinMax(data)
new_dead = findDeadSensors(data, minmax, meta['TaskId'], meta['IsPracticeTest'])
old_dead = validate.findDeadSensor(validate.findMinMax(data), isClipTask(filename))
if filename[-5:]=='2.txt':
print filename,old_dead, new_dead
time.sleep(0.1)
except Exception as e:
print e
print filename
'''
# <codecell>
biglist = myS3.getFilesBetween(datetime.utcnow()-timedelta(days=160), datetime.utcnow(), bucket, True)
print len(biglist)
# <codecell>
def lists_contain_same(lis1, lis2):
if lis1 == lis2:
return True
if len(lis1)!=len(lis2):
return False
for item in lis1:
if item not in lis2:
return False
for item in lis2:
if item not in lis1:
return False
bucket = conn.get_bucket('incoming-simscore-org')
theforgotten = myS3.getLeftBehind(daysback=7, conn=conn, sdb_domain=sdb_domain)
print theforgotten
if len(theforgotten) > 0:
mySQS.append_list_to_queue(theforgotten, comq)
# <codecell>
if mySQS.approx_total_messages(comq)==0:
print 'processing'
conn = boto.connect_s3(aws_ak, aws_sk)
bucket = conn.get_bucket('incoming-simscore-org')
t0 = datetime.now()-timedelta(days=178) #177
filelist = myS3.getFilesBetween(mindate=t0, maxdate=datetime.utcnow(), bucket=bucket, onlyTxtFiles=True)
#filelist = ['edge11/2013/01/07.18.32.15.109.0.txt','edge7/2013/01/07.16.59.48.370.2.txt','edge11/2013/01/04.21.36.02.109.0.txt']
print len(filelist), filelist
if len(filelist) > 0:
mySQS.append_list_to_queue(filelist, comq)
# <codecell>
comq.clear()
# <codecell>