def fetch_by_timestamps(table, model, timestamp_field, parsing_func, USERS_PER_REQUEST=40):
global START_TIME
session = db.session
all_found = False
while not all_found:
if START_TIME is None:
start = "2000-01-01T01:00:00.000000"
timestamp = getattr(model, timestamp_field)
last_user = session.query(model).order_by(db.desc(timestamp)).first()
if last_user is not None:
start = getattr(last_user, timestamp_field).isoformat()
START_TIME = start
print "Fetching", table, "starting from:", START_TIME
params = {'table': table,
'start_time': START_TIME,
'limit': USERS_PER_REQUEST
}
data = fetch(params)
if len(data) == 0:
break
last_time = parsing_func(data, session)
if last_time is None:
break
START_TIME = last_time.isoformat()
print "Completed fetching", table
def get_historical_metadata(resource_id, version=None):
query = ResourceVersion.query.filter_by(resource_id=resource_id). \
order_by(db.desc('version'))
if version is not None:
query = query.filter(ResourceVersion.version <= version)
query = query.limit(1000)
return query.all()
def get_content_for_version(resource_id, version):
if version == 0:
return ''
# get most recent full save
prev_string = ''
prev_version = 0
full_save = ScriptData.query.filter_by(resource_id=resource_id). \
filter(ScriptData.version <= version). \
order_by(db.desc('version')).first()
if full_save:
prev_string = full_save.data
prev_version = full_save.version
elif DuplicateScript.has_parent(resource_id):
dup = DuplicateScript.query. \
filter_by(new_script=resource_id).first()
if dup.from_version <= version:
prev_string = ScriptData.get_content_for_version(dup.from_script,
dup.from_version)
prev_version = dup.from_version
else:
return ScriptData.get_content_for_version(dup.from_script, version)
saves = ResourceVersion.query.filter_by(resource_id=resource_id).\
filter(ResourceVersion.version > prev_version). \
filter(ResourceVersion.version <= version). \
order_by('version').options(db.subqueryload('ops')).all()
for save in saves:
prev_string = ScriptData.apply_ops(prev_string, save.ops)
return prev_string
def get_latest_version_number(resource_id):
latest = ScriptData.query.filter_by(resource_id=resource_id). \
with_entities(ScriptData.version). \
order_by(db.desc('version')).first()
return latest[0]
def get_latest_version(resource_id):
latest = ResourceVersion.query.filter_by(resource_id=resource_id). \
order_by(db.desc('version')).first()
return latest
