예제 #1
0
import boto, time, json, pprint
from datetime import datetime, timedelta
import numpy as np

import fetch.myS3 as myS3
import fetch.mySQS as mySQS
import Simscore.validity_metrics as vm
from fetch.aws import aws_ak, aws_sk

# <codecell>

conn = boto.connect_s3(aws_ak, aws_sk)
bucket = conn.get_bucket('incoming-simscore-org')
mind = mindate=datetime.utcnow()-timedelta(days=6)
print mind, datetime.utcnow()
allfiles = myS3.getFilesBetween(mindate=mind, bucket=bucket, onlyTxtFiles=True)

# <codecell>

userfiles = []
for f in allfiles:
    edge = f.split('/')[0][4:]
    if f.split('.')[-3] != '109' and edge not in ['0','11','12']:
        userfiles.append(f)
        
ffs = myS3.getTestFiles(userfiles, bucket) 
print ffs

# <codecell>

report = {}
예제 #2
0
        minmax = validate.findMinMax(data)
        
        new_dead = findDeadSensors(data, minmax, meta['TaskId'], meta['IsPracticeTest'])
        old_dead = validate.findDeadSensor(validate.findMinMax(data), isClipTask(filename))
        
        if filename[-5:]=='2.txt':
            print filename,old_dead, new_dead
        time.sleep(0.1)
    except Exception as e:
        print e
        print filename
'''

# <codecell>

biglist = myS3.getFilesBetween(datetime.utcnow()-timedelta(days=160), datetime.utcnow(), bucket, True)
print len(biglist)

# <codecell>

def lists_contain_same(lis1, lis2):
    if lis1 == lis2:
        return True
    if len(lis1)!=len(lis2):
        return False
    for item in lis1:
        if item not in lis2:
            return False
    for item in lis2:
        if item not in lis1:
            return False
예제 #3
0
    bucket = conn.get_bucket('incoming-simscore-org')
    theforgotten = myS3.getLeftBehind(daysback=7, conn=conn, sdb_domain=sdb_domain)
    print theforgotten
    
    if len(theforgotten) > 0:
        mySQS.append_list_to_queue(theforgotten, comq)

# <codecell>

if mySQS.approx_total_messages(comq)==0:
    print 'processing'
    conn = boto.connect_s3(aws_ak, aws_sk)
    bucket = conn.get_bucket('incoming-simscore-org')
    
    t0 = datetime.now()-timedelta(days=178) #177
    filelist = myS3.getFilesBetween(mindate=t0, maxdate=datetime.utcnow(), bucket=bucket, onlyTxtFiles=True)
    #filelist =  ['edge11/2013/01/07.18.32.15.109.0.txt','edge7/2013/01/07.16.59.48.370.2.txt','edge11/2013/01/04.21.36.02.109.0.txt']
    print len(filelist), filelist
    
    if len(filelist) > 0:
        mySQS.append_list_to_queue(filelist, comq)
     

# <codecell>

comq.clear()

# <codecell>