def accede(config_file, time_series):
config = get_config(config_file, time_series)
logging.info('parsed config file %s:%s' % (config_file, time_series))
fx = IfcbFixity(config.psql_connect)
feed = IfcbFeed(config.psql_connect)
try:
year_pattern = config.year_pattern
except:
year_pattern = '....'
with xa(config.psql_connect) as (c, db):
for s in list_new_filesets(time_series,config.psql_connect,config.resolver,year_pattern=year_pattern): # FIXME hardcoded
try:
check_integrity(s.pid, s.hdr_path, s.adc_path, s.roi_path, s.schema_version)
except Exception, e:
logger.warn('%s FAIL integrity checks: %s' % (s.pid, e))
continue
# hot diggity, we've got some good data
# compute fixity
try:
fx.fix(s.pid, s.hdr_path, cursor=db, filetype='hdr')
logger.info('%s FIXITY computed for %s' % (s.pid, s.hdr_path))
fx.fix(s.pid, s.adc_path, cursor=db, filetype='adc')
logger.info('%s FIXITY computed for %s' % (s.pid, s.adc_path))
fx.fix(s.pid, s.roi_path, cursor=db, filetype='roi')
logger.info('%s FIXITY computed for %s' % (s.pid, s.roi_path))
except:
logger.error('%s FAIL fixity cannot be computed!' % s.pid)
c.rollback()
continue
# register bin
try:
ts = text2utcdatetime(s.date, s.date_format)
feed.create(s.pid, ts, cursor=db)
c.commit()
logger.info('%s DONE' % s.pid)
except:
logger.error('%s FAILED' % s.pid)
continue
def accession_demo(session,ts_label,root):
# now accede
for fs in get_resolver().ifcb.files.list_raw_filesets(root):
lid = fs['lid']
try:
parsed = parse_pid(lid)
except:
print 'barf %s' % lid
raise
ts = text2utcdatetime(parsed['timestamp'], parsed['timestamp_format'])
b = Bin(ts_label=ts_label, lid=lid, sample_time=ts)
session.add(b)
# now make mostly bogus fixity entries
now = datetime.now()
paths = [fs['hdr_path'], fs['adc_path'], fs['roi_path']]
filetypes = ['hdr','adc','roi']
for path,filetype in zip(paths,filetypes):
length = os.stat(path).st_size
name = os.path.basename(path)
#checksum = sha1_file(path)
checksum = 'placeholder'
f = File(local_path=path, filename=name, length=length, filetype=filetype, sha1=checksum, fix_time=now)
b.files.append(f)
session.commit()
def get_timestamp(parsed_pid):
"""extract the timestamp from a parsed pid and return it as a UTC datetime"""
return text2utcdatetime(parsed_pid[TIMESTAMP], parsed_pid[TIMESTAMP_FORMAT])
with xa(config.psql_connect) as (c, db):
for s in list_new_filesets(time_series,config.psql_connect,config.resolver,after_year=2005): # FIXME hardcoded
try:
check_integrity(s.pid, s.hdr_path, s.adc_path, s.roi_path, s.schema_version)
except Exception, e:
logging.warn('%s FAIL integrity checks: %s' % (s.pid, e))
continue
# hot diggity, we've got some good data
# compute fixity
try:
fx.fix(s.pid, s.hdr_path, cursor=db, filetype='hdr')
logging.info('%s FIXITY computed for %s' % (s.pid, s.hdr_path))
fx.fix(s.pid, s.adc_path, cursor=db, filetype='adc')
logging.info('%s FIXITY computed for %s' % (s.pid, s.adc_path))
fx.fix(s.pid, s.roi_path, cursor=db, filetype='roi')
logging.info('%s FIXITY computed for %s' % (s.pid, s.roi_path))
except:
logging.error('%s FAIL fixity cannot be computed!' % s.pid)
c.rollback()
continue
# register bin
try:
ts = text2utcdatetime(s.date, s.date_format)
feed.create(s.pid, ts, cursor=db)
c.commit()
logging.info('%s DONE' % s.pid)
except:
logging.error('%s FAILED' % s.pid)
continue
def nearest_demo(feed):
ts = '2013-09-10T13:45:22Z'
somedate = text2utcdatetime(ts)
n = 5
print '%d nearest bins to %s' % (n,ts)
print feed.nearest(n,somedate)
import sqlalchemy as sqla
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, BigInteger, String, DateTime, Boolean, ForeignKey
from sqlalchemy.orm import sessionmaker, relationship, backref
from sqlalchemy import and_, or_, desc
from sqlalchemy.sql.expression import func
from oii.times import text2utcdatetime, datetime2utcdatetime
from oii.utils import sha1_file
from oii.ifcb2 import get_resolver
from oii.ifcb2.identifiers import parse_pid
from oii.ifcb2.orm import Base, TimeSeries, DataDirectory, Bin, File, User
from oii.ifcb2.feed import Feed
from oii.ifcb2.accession import fast_accession
demo_end_time = text2utcdatetime('2013-09-20T00:00:00Z')
demo_start_time = demo_end_time - timedelta(hours=2)
# time range demo
def time_range_demo(feed):
print '2 hours worth of bins:'
for instance in feed.time_range(demo_start_time, demo_end_time):
print instance
# now extend this to show files
def files_demo(feed):
print '2 hours worth of files:'
for instance in feed.time_range(demo_start_time, demo_end_time):
print instance.files
