def fetch_by_timestamps(table, model, timestamp_field, parsing_func, USERS_PER_REQUEST=40): global START_TIME session = db.session all_found = False while not all_found: if START_TIME is None: start = "2000-01-01T01:00:00.000000" timestamp = getattr(model, timestamp_field) last_user = session.query(model).order_by(db.desc(timestamp)).first() if last_user is not None: start = getattr(last_user, timestamp_field).isoformat() START_TIME = start print "Fetching", table, "starting from:", START_TIME params = {'table': table, 'start_time': START_TIME, 'limit': USERS_PER_REQUEST } data = fetch(params) if len(data) == 0: break last_time = parsing_func(data, session) if last_time is None: break START_TIME = last_time.isoformat() print "Completed fetching", table
def get_historical_metadata(resource_id, version=None): query = ResourceVersion.query.filter_by(resource_id=resource_id). \ order_by(db.desc('version')) if version is not None: query = query.filter(ResourceVersion.version <= version) query = query.limit(1000) return query.all()
def get_content_for_version(resource_id, version): if version == 0: return '' # get most recent full save prev_string = '' prev_version = 0 full_save = ScriptData.query.filter_by(resource_id=resource_id). \ filter(ScriptData.version <= version). \ order_by(db.desc('version')).first() if full_save: prev_string = full_save.data prev_version = full_save.version elif DuplicateScript.has_parent(resource_id): dup = DuplicateScript.query. \ filter_by(new_script=resource_id).first() if dup.from_version <= version: prev_string = ScriptData.get_content_for_version(dup.from_script, dup.from_version) prev_version = dup.from_version else: return ScriptData.get_content_for_version(dup.from_script, version) saves = ResourceVersion.query.filter_by(resource_id=resource_id).\ filter(ResourceVersion.version > prev_version). \ filter(ResourceVersion.version <= version). \ order_by('version').options(db.subqueryload('ops')).all() for save in saves: prev_string = ScriptData.apply_ops(prev_string, save.ops) return prev_string
def get_latest_version_number(resource_id): latest = ScriptData.query.filter_by(resource_id=resource_id). \ with_entities(ScriptData.version). \ order_by(db.desc('version')).first() return latest[0]
def get_latest_version(resource_id): latest = ResourceVersion.query.filter_by(resource_id=resource_id). \ order_by(db.desc('version')).first() return latest