def import_obj(cls, slc_to_import, slc_to_override, import_time=None):
"""Inserts or overrides slc in the database.
remote_id and import_time fields in params_dict are set to track the
slice origin and ensure correct overrides for multiple imports.
Slice.perm is used to find the datasources and connect them.
:param Slice slc_to_import: Slice object to import
:param Slice slc_to_override: Slice to replace, id matches remote_id
:returns: The resulting id for the imported slice
:rtype: int
"""
session = db.session
make_transient(slc_to_import)
slc_to_import.dashboards = []
slc_to_import.alter_params(
remote_id=slc_to_import.id, import_time=import_time)
slc_to_import = slc_to_import.copy()
params = slc_to_import.params_dict
slc_to_import.datasource_id = ConnectorRegistry.get_datasource_by_name(
session, slc_to_import.datasource_type, params['datasource_name'],
params['schema'], params['database_name']).id
if slc_to_override:
slc_to_override.override(slc_to_import)
session.flush()
return slc_to_override.id
session.add(slc_to_import)
logging.info('Final slice: {}'.format(slc_to_import.to_json()))
session.flush()
return slc_to_import.id
def cash_instances_to_dict(self, list_of_classes, do_make_transient = False):
#Закрываем сессию. Открываем новую и считываем в нее все классы. Отвязываем их от сессии.
#Возвращаем список инстансов в виде словаря. Надеюсь, это поможет работать с ними сколь угодно много..
#Была идея оставить возможность не закрывать сесиию - отказался. В худшем случае, можно отдельную сессию создавать.
#Но две одновременные сессии - тоже опасно.
self.close_session()
self.private_activate_session()
dict_with_instances = dict()
for cls_i in list_of_classes: #Интересно, нужно ли как-то особо считывать взаимосвязи
repr_cls_i = with_polymorphic(cls_i, '*')
inst_list = []
for inst_i in self.active_session.query(repr_cls_i).options(immediateload('*')).all():
#if not(inst_i in inst_list):
inst_list.append(inst_i)
dict_with_instances[cls_i.__name__] = inst_list
self.active_session.expunge_all() #именно поэтому закрываем сессию до запуска
for inst_list in dict_with_instances.itervalues():
for inst_i in inst_list:
if hasattr(inst_i, "disconnected_from_session"):
raise BaseException("[c_session_handler][cash_instances_to_dict] you cannot use 'disconnected_from_session' attribute in a class here")
inst_i.disconnected_from_session = True
if do_make_transient: #Без этого может пытаться обратиться к базе данных
make_transient(inst_i)
self.close_session()
return dict_with_instances
def _copy(self):
schema = NavigationCopySchema().bind(request=self.request)
try:
controls = schema.deserialize(self.request.params)
(
DBSession.query(Navigation)
.filter(
Navigation.condition_position_id(
controls.get('position_id')
)
)
.delete()
)
navigations_from = (
DBSession.query(Navigation)
.filter(
Navigation.condition_position_id(
controls.get('from_position_id')
)
)
)
for navigation in navigations_from:
make_transient(navigation)
navigation.id = None
navigation.position_id = controls.get('position_id')
DBSession.add(navigation)
return {'success_message': _(u'Copied')}
except colander.Invalid, e:
return {
'error_message': _(u'Please, check errors'),
'errors': e.asdict()
}
def copy_from_position(source_position_id, target_position_id):
assert isinstance(source_position_id, int), \
u"Integer expected"
assert isinstance(target_position_id, int), \
u"Integer expected"
(
DBSession.query(Permision)
.filter(
Permision.condition_position_id(target_position_id)
)
.delete()
)
permisions_from = (
DBSession.query(Permision)
.filter(
Permision.condition_position_id(source_position_id)
)
)
for permision in permisions_from:
make_transient(permision)
permision.id = None
permision.position_id = target_position_id
DBSession.add(permision)
return
def run(self):
if statsd:
statsd.incr(self.__class__.__name__.lower()+'_start', 1, 1)
# Adding an entry in the DB
session = settings.Session()
self.state = State.RUNNING
if not self.id:
session.add(self)
else:
session.merge(self)
session.commit()
id_ = self.id
make_transient(self)
self.id = id_
# Run
self._execute()
#better to submit tasks and poll for completion...
# Marking the success in the DB
self.end_date = datetime.now()
self.state = State.SUCCESS
session.merge(self)
session.commit()
session.close()
if statsd:
statsd.incr(self.__class__.__name__.lower()+'_end', 1, 1)
def test_import_dashboard_1_slice(self):
slc = self.create_slice('health_slc', id=10006)
dash_with_1_slice = self.create_dashboard(
'dash_with_1_slice', slcs=[slc], id=10002)
dash_with_1_slice.position_json = """
{{"DASHBOARD_VERSION_KEY": "v2",
"DASHBOARD_CHART_TYPE-{0}": {{
"type": "DASHBOARD_CHART_TYPE",
"id": {0},
"children": [],
"meta": {{
"width": 4,
"height": 50,
"chartId": {0}
}}
}}
}}
""".format(slc.id)
imported_dash_id = models.Dashboard.import_obj(
dash_with_1_slice, import_time=1990)
imported_dash = self.get_dash(imported_dash_id)
expected_dash = self.create_dashboard(
'dash_with_1_slice', slcs=[slc], id=10002)
make_transient(expected_dash)
self.assert_dash_equals(
expected_dash, imported_dash, check_position=False)
self.assertEquals({'remote_id': 10002, 'import_time': 1990},
json.loads(imported_dash.json_metadata))
expected_position = dash_with_1_slice.position
self.assertEquals(expected_position, imported_dash.position)
def copy_notification(id):
notification = Notification.query.filter_by(id=id).first_or_404()
desc = notification.description
if notification.user != current_user and not current_user.is_admin():
abort(403)
notification.id = None
notification.description = desc + ' Clone'
make_transient(notification)
db.session.add(notification)
db.session.commit()
old_settings = NotificationSetting.query.filter_by(notification_id=id).all()
for s in old_settings:
s.id = None
s.notification_id = notification.id
make_transient(s)
db.session.add(s)
db.session.commit()
current_app.decoder.refresh_notifier(notification.id)
flash('Notification cloned.', 'success')
return redirect(url_for('notifications.index'))
def test_import_dashboard_1_slice(self):
slc = self.create_slice('health_slc', id=10006)
dash_with_1_slice = self.create_dashboard(
'dash_with_1_slice', slcs=[slc], id=10002)
dash_with_1_slice.position_json = """
[{{
"col": 5,
"row": 10,
"size_x": 4,
"size_y": 2,
"slice_id": "{}"
}}]
""".format(slc.id)
imported_dash_id = models.Dashboard.import_obj(
dash_with_1_slice, import_time=1990)
imported_dash = self.get_dash(imported_dash_id)
expected_dash = self.create_dashboard(
'dash_with_1_slice', slcs=[slc], id=10002)
make_transient(expected_dash)
self.assert_dash_equals(
expected_dash, imported_dash, check_position=False)
self.assertEquals({"remote_id": 10002, "import_time": 1990},
json.loads(imported_dash.json_metadata))
expected_position = dash_with_1_slice.position_array
expected_position[0]['slice_id'] = '{}'.format(
imported_dash.slices[0].id)
self.assertEquals(expected_position, imported_dash.position_array)
def import_obj(cls, slc_to_import, import_time=None):
"""Inserts or overrides slc in the database.
remote_id and import_time fields in params_dict are set to track the
slice origin and ensure correct overrides for multiple imports.
Slice.perm is used to find the datasources and connect them.
"""
session = db.session
make_transient(slc_to_import)
slc_to_import.dashboards = []
slc_to_import.alter_params(
remote_id=slc_to_import.id, import_time=import_time)
# find if the slice was already imported
slc_to_override = None
for slc in session.query(Slice).all():
if ('remote_id' in slc.params_dict and
slc.params_dict['remote_id'] == slc_to_import.id):
slc_to_override = slc
slc_to_import = slc_to_import.copy()
params = slc_to_import.params_dict
slc_to_import.datasource_id = ConnectorRegistry.get_datasource_by_name(
session, slc_to_import.datasource_type, params['datasource_name'],
params['schema'], params['database_name']).id
if slc_to_override:
slc_to_override.override(slc_to_import)
session.flush()
return slc_to_override.id
session.add(slc_to_import)
logging.info('Final slice: {}'.format(slc_to_import.to_json()))
session.flush()
return slc_to_import.id
def restore(request, coupon_id):
if coupon_id:
coupon_id = int(coupon_id)
if request.user.is_focus:
coupon = request.db_session.query(Coupon).filter_by(id=coupon_id, is_deleted=True).first()
else:
coupon = request.db_session.query(Coupon).filter_by(id=coupon_id, is_deleted=True, company_id=request.user.company_id).first()
try:
coupon.modified_by_id = request.user.id
request.db_session.commit()
request.db_session.expunge(coupon)
make_transient(coupon)
coupon.main_coupon_id = coupon.main_coupon_id if coupon.main_coupon_id else coupon.id
coupon.id = None
coupon.restore()
request.db_session.add(coupon)
request.db_session.commit()
response = json_response_content('success', 'Coupon was successfully restored!')
except Exception as e:
request.db_session.rollback()
response = json_response_content('error', 'There was an error during coupon restore: {0}'.format(str(e)))
return JsonResponse(response)
else:
raise Http404
def test_import_override_dashboard_2_slices(self):
e_slc = self.create_slice('e_slc', id=10009, table_name='energy_usage')
b_slc = self.create_slice('b_slc', id=10010, table_name='birth_names')
dash_to_import = self.create_dashboard(
'override_dashboard', slcs=[e_slc, b_slc], id=10004)
imported_dash_id_1 = models.Dashboard.import_obj(
dash_to_import, import_time=1992)
# create new instances of the slices
e_slc = self.create_slice(
'e_slc', id=10009, table_name='energy_usage')
b_slc = self.create_slice(
'b_slc', id=10010, table_name='birth_names')
c_slc = self.create_slice('c_slc', id=10011, table_name='birth_names')
dash_to_import_override = self.create_dashboard(
'override_dashboard_new', slcs=[e_slc, b_slc, c_slc], id=10004)
imported_dash_id_2 = models.Dashboard.import_obj(
dash_to_import_override, import_time=1992)
# override doesn't change the id
self.assertEquals(imported_dash_id_1, imported_dash_id_2)
expected_dash = self.create_dashboard(
'override_dashboard_new', slcs=[e_slc, b_slc, c_slc], id=10004)
make_transient(expected_dash)
imported_dash = self.get_dash(imported_dash_id_2)
self.assert_dash_equals(
expected_dash, imported_dash, check_position=False)
self.assertEquals({"remote_id": 10004, "import_time": 1992},
json.loads(imported_dash.json_metadata))
def test_transient_exception(self):
"""An object that goes from a pk value to transient/pending
doesn't count as a "pk" switch.
"""
users, Address, addresses, User = (self.tables.users,
self.classes.Address,
self.tables.addresses,
self.classes.User)
mapper(User, users)
mapper(Address, addresses, properties={'user':relationship(User)})
sess = create_session()
u1 = User(id=5, name='u1')
ad1 = Address(email_address='e1', user=u1)
sess.add_all([u1, ad1])
sess.flush()
make_transient(u1)
u1.id = None
u1.username='******'
sess.add(u1)
sess.flush()
eq_(ad1.user_id, 5)
sess.expire_all()
eq_(ad1.user_id, 5)
ne_(u1.id, 5)
ne_(u1.id, None)
eq_(sess.query(User).count(), 2)
def restore_provider_details(notify_db, notify_db_session):
"""
We view ProviderDetails as a static in notify_db_session, since we don't modify it... except we do, we updated
priority. This fixture is designed to be used in tests that will knowingly touch provider details, to restore them
to previous state.
Note: This doesn't technically require notify_db_session (only notify_db), but kept as a requirement to encourage
good usage - if you're modifying ProviderDetails' state then it's good to clear down the rest of the DB too
"""
existing_provider_details = ProviderDetails.query.all()
existing_provider_details_history = ProviderDetailsHistory.query.all()
# make transient removes the objects from the session - since we'll want to delete them later
for epd in existing_provider_details:
make_transient(epd)
for epdh in existing_provider_details_history:
make_transient(epdh)
yield
# also delete these as they depend on provider_details
ProviderRates.query.delete()
ProviderDetails.query.delete()
ProviderDetailsHistory.query.delete()
notify_db.session.commit()
notify_db.session.add_all(existing_provider_details)
notify_db.session.add_all(existing_provider_details_history)
notify_db.session.commit()
def update_scorer(scorer):
# detach from the session to load the database instance to compare
scorer_id = scorer.id
make_transient(scorer)
scorer.id = scorer_id
scorer_db = Scorer.query.get(scorer_id)
if scorer_db.scorer == scorer.scorer:
return
# reattach the instance
scorer = db.session.merge(scorer)
bets = BetScorer.query.filter(or_(BetScorer.scorer1==scorer, BetScorer.scorer2==scorer))
for bet in bets:
if not scorer.scorer and bet.score == 0:
continue
if scorer.scorer:
bet.score += SCORER_POINTS
else:
bet.score -= SCORER_POINTS
# pass as param since the user.bet_scorer backref is None here
update_total_score(bet.user, bet_scorer=bet)
db.session.commit()
def copy(self):
db.session.expunge(self)
make_transient(self)
self.id = None
db.session.add(self)
db.session.flush()
return self
def main():
with sessionmanager as session:
for x in session.query(NPriceList).filter_by(almacen_id=1):
session.expunge(x)
make_transient(x)
x.pid = None
x.almacen_id = 3
session.add(x)
def test_sched_a_fulltext_trigger(self):
# Test create
nml_row = self.NmlSchedAFactory(
rpt_yr=2014,
contbr_nm='Sheldon Adelson',
contb_receipt_dt=datetime.datetime(2014, 1, 1)
)
self.FItemReceiptOrExp(
sub_id=nml_row.sub_id,
rpt_yr=2014,
)
db.session.commit()
manage.refresh_itemized()
search = models.ScheduleA.query.filter(
models.ScheduleA.sub_id == nml_row.sub_id
).one()
self.assertEqual(search.contributor_name_text, "'adelson':2 'sheldon':1")
# Test update
nml_row.contbr_nm = 'Shelly Adelson'
db.session.add(nml_row)
db.session.commit()
manage.refresh_itemized()
search = models.ScheduleA.query.filter(
models.ScheduleA.sub_id == nml_row.sub_id
).one()
db.session.refresh(search)
self.assertEqual(search.contributor_name_text, "'adelson':2 'shelli':1")
# Test delete
db.session.delete(nml_row)
db.session.commit()
manage.refresh_itemized()
self.assertEqual(
models.ScheduleA.query.filter(
models.ScheduleA.sub_id == nml_row.sub_id
).count(),
0,
)
# Test sequential writes
make_transient(nml_row)
db.session.add(nml_row)
db.session.commit()
db.session.delete(nml_row)
db.session.commit()
make_transient(nml_row)
db.session.add(nml_row)
db.session.commit()
manage.refresh_itemized()
self.assertEqual(
models.ScheduleA.query.filter(
models.ScheduleA.sub_id == nml_row.sub_id
).count(),
1,
)
def save(self):
data = self.data
# remove the button
data.pop('submit', None)
forum = Forum(**data)
# flush SQLA info from created instance so that it can be merged
make_transient(forum)
make_transient_to_detached(forum)
return forum.save()
def test_sched_a_fulltext_trigger(self):
# Test create
row = self.SchedAFactory(
rpt_yr=2014,
contbr_nm='Sheldon Adelson',
load_date=datetime.datetime.now(),
sub_id=7,
)
db.session.commit()
db.session.execute('select update_aggregates()')
search = models.ScheduleA.query.filter(
models.ScheduleA.sched_a_sk == row.sched_a_sk
).one()
self.assertEqual(search.contributor_name_text, "'adelson':2 'sheldon':1")
# Test update
row.contbr_nm = 'Shelly Adelson'
db.session.add(row)
db.session.commit()
db.session.execute('select update_aggregates()')
search = models.ScheduleA.query.filter(
models.ScheduleA.sched_a_sk == row.sched_a_sk
).one()
db.session.refresh(search)
self.assertEqual(search.contributor_name_text, "'adelson':2 'shelli':1")
# Test delete
db.session.delete(row)
db.session.commit()
db.session.execute('select update_aggregates()')
self.assertEqual(
models.ScheduleA.query.filter(
models.ScheduleA.sched_a_sk == row.sched_a_sk
).count(),
0,
)
# Test sequential writes
make_transient(row)
db.session.add(row)
db.session.commit()
db.session.delete(row)
db.session.commit()
make_transient(row)
db.session.add(row)
db.session.commit()
db.session.execute('select update_aggregates()')
self.assertEqual(
models.ScheduleA.query.filter(
models.ScheduleA.sched_a_sk == row.sched_a_sk
).count(),
1,
)
def save(self):
data = self.data
# delete submit and csrf_token from data
data.pop('submit', None)
data.pop('csrf_token', None)
forum = Forum(**data)
# flush SQLA info from created instance so that it can be merged
make_transient(forum)
make_transient_to_detached(forum)
return forum.save()
def new_version(self, session):
# make us transient (removes persistent
# identity).
make_transient(self)
old_version = session.query(self.__class__).get(self.id)
self.valid_from = datetime.datetime.now()
old_version.valid_to = self.valid_from
# set 'id' to None.
# a new PK will be generated on INSERT.
self.id = None
def get_version(recipe_id, version_id):
valid_ids = [x[0] for x in get_versions(recipe_id)]
if version_id not in valid_ids:
raise BadVersionError
recipe = get_recipe(recipe_id)
db.session.expunge(recipe)
make_transient(recipe)
data = db.session.query(db.RecipeData).get(version_id)
db.session.expunge(data)
make_transient(data)
recipe.data = data
return recipe
def sync(self, other):
types = self.types & other.types
other_id = other.host_id
for name in types:
new_ver = max(self.local_version(name), other.local_version(name))+1
basever = self.get_version(other.host_id, name)
# hackedyhack. within this lies almost infinite potential for optimization.
for o in self.objects(name):
self.session.add(o.bump(new_ver))
for o in other.objects(name):
other.session.add(o.bump(new_ver))
self.bump_version(other.host_id, name, new_ver)
other.bump_version(self.host_id, name, new_ver)
new_ver += 1
self.session.commit()
other.session.commit()
local = { o._uuid: o for o in self.objects(name) }
local_uuids = set(local.keys())
remote = { o._uuid: o for o in other.objects(name) }
remote_uuids = set(remote.keys())
for uuid in local_uuids | remote_uuids:
if uuid not in remote_uuids:
o = local[uuid]
make_transient(o)
other._sync_put(o)
elif uuid not in local_uuids:
o = remote[uuid]
make_transient(o)
self._sync_put(o)
else: # obj in local and remote
lo, ro, base = local[uuid], remote[uuid], self.find_base(name, uuid, basever)
if lo._updated != ro._updated:
lot, rot = lo.prepare_merge(ro, base), ro.prepare_merge(lo, base)
lo.apply_merge(lot)
ro.apply_merge(rot)
self._sync_put(lo)
other._sync_put(ro)
self.session.commit()
other.session.commit()
# hackedyhack. within this lies almost infinite potential for optimization.
for o in self.objects(name):
self.session.add(o.bump(new_ver))
for o in other.objects(name):
other.session.add(o.bump(new_ver))
self.session.commit()
other.session.commit()
def disconnect_from_db(docs):
'''Make sure that docs won't synchronise with the database if changed.'''
for d in docs:
#make_transient removes the primary keys, hack to back them up and
#reset them after
id_attrs = [a.name for a in class_mapper(d.__class__).primary_key]
id_vals = {a: getattr(d, a) for a in id_attrs}
session.make_transient(d)
for attr, val in id_vals.iteritems():
setattr(d, attr, val)
def clone_a_contract(contract):
'''Takes a contract object and clones it
The clone strips the following properties:
+ Financial ID
+ Expiration Date
+ Assigned To
+ Current Stage
+ Contract HREF
Relationships are handled as follows:
+ Stars, Follows - moved to new contract (dropped from old)
+ Stage, Flow - Duplicated
+ Properties, Notes, Line Items, Companies kept on old
'''
old_contract_id = int(contract.id)
subscribers = [
('follow', list(contract.followers)),
('star', list(contract.starred))
]
db.session.expunge(contract)
make_transient(contract)
contract.id = None
contract.financial_id = None
contract.expiration_date = None
contract.assigned_to = None
contract.current_stage = None
contract.contract_href = None
old_contract = get_one_contract(old_contract_id)
# group everything that will rebuild the trigger
# into one flush
db.session.add(contract)
old_contract.is_archived = True
old_contract.description = old_contract.description + ' [Archived]'
# we have to commit here in order to manage the relationships
db.session.commit()
for interaction, users in subscribers:
for i in users:
unfollow_a_contract(old_contract_id, i, interaction)
follow_a_contract(contract.id, i, interaction)
db.session.commit()
# set the parent
contract.parent_id = old_contract_id
db.session.commit()
return contract
def delay_feedback(self, submission):
if submission is None or submission.score is None:
return submission
due_date = submission.question.homework.due_date
submission.item_responses # instantiate item responses before we detach object from session
make_transient(submission) # detaches SQLAlchemy object from session
now = pdt_now()
time_available = min(submission.time + timedelta(minutes=30), due_date)
if now < time_available:
submission.score = None
submission.comments = '''Feedback on your submission will be available in %s minutes, at %s. Please refresh the page at that time to view it.''' % (1 + (time_available - now).seconds // 60, time_available.strftime("%H:%M"))
return submission
def import_datasource(
session,
i_datasource,
lookup_database,
lookup_datasource,
import_time):
"""Imports the datasource from the object to the database.
Metrics and columns and datasource will be overrided if exists.
This function can be used to import/export dashboards between multiple
superset instances. Audit metadata isn't copies over.
"""
make_transient(i_datasource)
logging.info('Started import of the datasource: {}'.format(
i_datasource.to_json()))
i_datasource.id = None
i_datasource.database_id = lookup_database(i_datasource).id
i_datasource.alter_params(import_time=import_time)
# override the datasource
datasource = lookup_datasource(i_datasource)
if datasource:
datasource.override(i_datasource)
session.flush()
else:
datasource = i_datasource.copy()
session.add(datasource)
session.flush()
for m in i_datasource.metrics:
new_m = m.copy()
new_m.table_id = datasource.id
logging.info('Importing metric {} from the datasource: {}'.format(
new_m.to_json(), i_datasource.full_name))
imported_m = i_datasource.metric_cls.import_obj(new_m)
if (imported_m.metric_name not in
[m.metric_name for m in datasource.metrics]):
datasource.metrics.append(imported_m)
for c in i_datasource.columns:
new_c = c.copy()
new_c.table_id = datasource.id
logging.info('Importing column {} from the datasource: {}'.format(
new_c.to_json(), i_datasource.full_name))
imported_c = i_datasource.column_cls.import_obj(new_c)
if (imported_c.column_name not in
[c.column_name for c in datasource.columns]):
datasource.columns.append(imported_c)
session.flush()
return datasource.id
def delete(self, defer_commit=False):
"""
Method to either Hard or Soft Delete a row.
Params
:param defer_commit: boolean
Return
:return: self
"""
db.session.delete(self)
if not defer_commit:
db.session.commit()
make_transient(self)
return self
def customDuplicate(patch_id):
if not localAdmin() and not adminRole():
log_Error("{} does not have permission to duplicate custom patch.".format(session.get('user')))
return json.dumps({'data': {}}, default=json_serial), 403
qGet1 = MpPatch.query.filter(MpPatch.puuid == patch_id).first()
if qGet1 is not None:
make_transient(qGet1)
_new_puuid = str(uuid.uuid4())
_new_patch_name = "Copy_"+qGet1.patch_name
qGet1.rid = None
setattr(qGet1, 'puuid', _new_puuid)
setattr(qGet1, 'patch_name', _new_patch_name)
setattr(qGet1, 'patch_state', "Create")
setattr(qGet1, 'active', "0")
_pkg_path = qGet1.pkg_path
_pkg_url = qGet1.pkg_url
setattr(qGet1, 'pkg_path', _pkg_path.replace(patch_id, _new_puuid))
setattr(qGet1, 'pkg_url', _pkg_url.replace(patch_id, _new_puuid))
db.session.add(qGet1)
# Duplicate patch criteria
qGet2 = MpPatchesCriteria.query.filter(MpPatchesCriteria.puuid == patch_id).all()
if qGet2 is not None:
for x in qGet2:
make_transient(x)
x.rid = None
setattr(x, 'puuid', _new_puuid)
db.session.add(x)
# Need to duplicate patch on file system
try:
_patch_dir0 = "/opt/MacPatch/Content/Web/patches/" + patch_id
if os.path.exists(_patch_dir0):
_patch_dir1 = "/opt/MacPatch/Content/Web/patches/" + _new_puuid
copytree(_patch_dir0,_patch_dir1)
else:
log("{}, unable to duplicate custom patch {}({}). Directory does not exist".format(session.get('user'), qGet1.patch_name, patch_id))
except OSError, e:
log_Error("Error: %s - %s." % (e.filename,e.strerror))
log("{} duplicate custom patch {}({}).".format(session.get('user'), qGet1.patch_name, patch_id))
def copy_tables():
""" Copies a few items from each table into a test database
Should not be called in the same session after reset_test_db(); you will get
a mapper error for some reason. Instead, call reset_test_db(), then close
the python session, then, in a new session, call copy_tables(), to update
the tables for a schema change. """
toy = get_db_session()
db = models.get_db_session()
logger.info("Populating test tables")
businesses = db.query(Business).order_by(Business.id)[0:5]
locations = [b.location for b in businesses]
# [b.categories ...] is a list of lists, so we need a little more processing
categories = set().union(*[b.categories for b in businesses])
tweets = db.query(Tweet).order_by(Tweet.id)[0:5]
reviews = []
for b in businesses:
reviews.extend(
db.query(YelpReview).filter(YelpReview.business_id == b.id)[0:5]
)
documents = [r.document for r in reviews] + [t.document for t in tweets]
doc_assoc = [r.document_rel for r in reviews] + \
[t.document_rel for t in tweets]
tables = [businesses, locations, categories, reviews, tweets, \
documents, doc_assoc]
# detach all objects from db session before putting them in toy
for t in tables:
for obj in t: make_transient(obj)
# only after *everything* is transient do we add anything
for t in tables: toy.add_all(t)
# in addition we add the junction table for business categories
b_ids = [b.id for b in businesses]
business_cat = db.execute(business_category_table.select().
where(business_category_table.c.business_id.in_(
[b.id for b in businesses]
)))
for row in business_cat:
toy.execute(business_category_table.insert(), row)
toy.commit()
def copy_bottle(self):
make_transient(self)
self.id = None
db.session.add(self)
db.session.commit()
def _task_instances_for_dag_run(self, dag_run, session=None):
"""
Returns a map of task instance key to task instance object for the tasks to
run in the given dag run.
:param dag_run: the dag run to get the tasks from
:type dag_run: models.DagRun
:param session: the database session object
:type session: Session
"""
tasks_to_run = {}
if dag_run is None:
return tasks_to_run
# check if we have orphaned tasks
if AIRFLOW_VERSION_2:
reset_state_for_orphaned_tasks(
self, filter_by_dag_run=dag_run, session=session
)
else:
self.reset_state_for_orphaned_tasks(
filter_by_dag_run=dag_run, session=session
)
# for some reason if we don't refresh the reference to run is lost
dag_run.refresh_from_db()
make_transient(dag_run)
# DBNDPATCH
# implements batch update
session.query(TI).filter(
TI.dag_id == self.dag_id,
TI.execution_date == self.execution_date,
TI.state == State.NONE,
).update(
{
TI.state: State.SCHEDULED,
TI.start_date: timezone.utcnow(),
TI.end_date: timezone.utcnow(),
},
synchronize_session="fetch",
)
# TODO(edgarRd): AIRFLOW-1464 change to batch query to improve perf
#
task_instances = dag_run.get_task_instances()
for ti in task_instances:
# all tasks part of the backfill are scheduled to run
if ti.state == State.NONE:
# no waiting for the airflow - batch upate
# ti.set_state(State.SCHEDULED, session=session)
ti.state = State.SCHEDULED
ti.start_date = timezone.utcnow()
ti.end_date = timezone.utcnow()
session.merge(ti)
if ti.state != State.REMOVED:
ti._log = logging.getLogger("airflow.task")
tasks_to_run[ti.key] = ti
session.commit()
return tasks_to_run
def make_copy(model: BaseModel) -> None:
make_transient(model)
model.id = None
model.created_at = None
def create_admin_video(admin_name="Max", video_name="jacksonhole.mp4"):
check_box_DB_segment = TinyDB("check_box_db_segment.json")
#get annotation info
annotations = get_annotations()
annotation = annotations[video_name]
#get admin info
admin = session.query(User).filter(User.username == admin_name).first()
admin_id = admin.id
admin_name = admin.username
#max id: primary key duplication issue
max_video_id = session.query(func.max(Video.id).label("max_id")).one()
#create video object for admin
sample_video = session.query(Video).filter(
Video.slug == annotation.workers[0] + '_' + video_name).first()
sample_labels = session.query(Label).filter(
Label.videoid == sample_video.id).all()
labels = ""
for sample_label in sample_labels:
labels = labels + sample_label.text + " "
load_admin_video(video_name, labels)
session.commit()
target_video = session.query(Video).filter(Video.slug == admin_name + '_' +
video_name).first()
query = Query()
segments = check_box_DB_segment.search(query.videoname == video_name)
#create segment object for admin
for segment in segments:
#print(1)
sample_segment = session.query(Segment).filter(
Segment.id == segment["segment_id"]).first()
target_segment = session.query(Segment).filter(
Segment.videoid == target_video.id,
Segment.start == sample_segment.start).first()
jobs = session.query(Job).filter(
Job.segmentid == segment["segment_id"]).all()
for job in jobs:
target_job = session.query(Job).filter(
Job.segmentid == target_segment.id).first()
paths = session.query(Path).filter(Path.jobid == job.id).all()
#print("job")
#print(temp_jobid)
for path in paths:
temp_pathid = path.id
session.expunge(path)
make_transient(path)
max_path_id = session.query(func.max(
Path.id).label("max_id")).one()
path.id = max_path_id.max_id + 1
path.jobid = target_job.id
label_text = session.query(Label).filter(
Label.id == path.labelid).first().text
label = session.query(Label).filter(
Label.videoid == target_video.id,
Label.text == label_text).first().id
path.labelid = label
new_path = Path(id=max_path_id.max_id + 1,
jobid=target_job.id,
labelid=label)
session.add(new_path)
session.commit()
boxes = session.query(Box).filter(
Box.pathid == temp_pathid).all()
for box in boxes:
print(box.frame)
session.expunge(box)
make_transient(box)
max_box_id = session.query(
func.max(Box.id).label("max_id")).one()
box.id = max_box_id.max_id + 1
box.pathid = path.id
#print(path.id)
session.add(box)
session.commit()
def process_dag(self, dag, queue):
"""
This method schedules a single DAG by looking at the latest
run for each task and attempting to schedule the following run.
As multiple schedulers may be running for redundancy, this
function takes a lock on the DAG and timestamps the last run
in ``last_scheduler_run``.
"""
DagModel = models.DagModel
session = settings.Session()
# picklin'
pickle_id = None
if self.do_pickle and self.executor.__class__ not in (
executors.LocalExecutor, executors.SequentialExecutor):
pickle_id = dag.pickle(session).id
# obtain db lock
db_dag = session.query(DagModel).filter_by(
dag_id=dag.dag_id).with_for_update().one()
last_scheduler_run = db_dag.last_scheduler_run or datetime(2000, 1, 1)
secs_since_last = (datetime.now() - last_scheduler_run).total_seconds()
if secs_since_last < self.heartrate:
# release db lock
session.commit()
session.close()
return None
# Release the db lock
# the assumption here is that process_dag will take less
# time than self.heartrate otherwise we might unlock too
# quickly and this should moved below, but that would increase
# the time the record is locked and is blocking for other calls.
db_dag.last_scheduler_run = datetime.now()
session.commit()
# update the state of the previously active dag runs
dag_runs = DagRun.find(dag_id=dag.dag_id,
state=State.RUNNING,
session=session)
active_dag_runs = []
for run in dag_runs:
# do not consider runs that are executed in the future
if run.execution_date > datetime.now():
continue
# todo: run.dag is transient but needs to be set
run.dag = dag
# todo: preferably the integrity check happens at dag collection time
run.verify_integrity(session=session)
run.update_state(session=session)
if run.state == State.RUNNING:
make_transient(run)
active_dag_runs.append(run)
for run in active_dag_runs:
# this needs a fresh session sometimes tis get detached
tis = run.get_task_instances(state=(State.NONE,
State.UP_FOR_RETRY))
# this loop is quite slow as it uses are_dependencies_met for
# every task (in ti.is_runnable). This is also called in
# update_state above which has already checked these tasks
for ti in tis:
task = dag.get_task(ti.task_id)
# fixme: ti.task is transient but needs to be set
ti.task = task
# future: remove adhoc
if task.adhoc:
continue
if ti.is_runnable(flag_upstream_failed=True):
self.logger.debug('Queuing task: {}'.format(ti))
queue.put((ti.key, pickle_id))
session.close()
def clone_gp(self, version='latest'):
global logger
logger.info("Cloning GenomeProject {}, version: {}".format(
self.gpv_id, version))
session = fetch_session()
try:
# See if we have a GenomeProject_Meta for this GP
logger.debug("Attempting to clone metadata")
gp_meta = session.query(GenomeProject_Meta).filter(
GenomeProject_Meta.gpv_id == self.gpv_id).first()
old_path = self.gpv_directory
old_gpv_id = self.gpv_id
# Remove the GP object from the session and
# unlink it's primary key
logger.debug("Expunging ourself from the session")
session.expunge(self)
make_transient(self)
self.gpv_id = None
# Update the session with the new version,
# add the GP back to the session and commit it
version = Version.fetch(version)
self.version_id = version
self.gpv_directory = os.path.join(
Version.fetch_path(version), self.genome_name,
self.assembly_accession + '_' + self.asm_name)
# We've saved the object, make the file system symlink
# but first we have to check if we point to another base object,
# if so find that path and link to it
if self.prev_gpv:
root_gp = session.query(GenomeProject).filter(
GenomeProject.gpv_id == self.prev_gpv).first()
if not root_gp:
logger.critical(
"We think we should have a root gpv_id {} but we can't find it, time to freak out"
.format(self.prev_gpv))
raise Exception(
"We can't find gpv_id {} but we ({}) seem to point to it"
.format(self.prev_gpv, self.gpv_id))
# Now remember the path for the root GP so we can symlink to it
old_path = root_gp.gpv_directory
if os.path.exists(old_path) and self.verify_basedir():
logger.debug("Making symlink from {} to {}".format(
old_path, self.gpv_directory))
os.symlink(old_path, self.gpv_directory)
else:
logger.error(
"We couldn't find the old path {} to make the symlink from, this is a problem"
.format(old_path))
logger.debug("Committing self")
session.add(self)
session.commit()
# If we have a metadata object and we've successfully
# updated ourself, clone the gp_meta object
if gp_meta:
logger.debug("We have metadata, clone: {}".format(gp_meta))
gp_meta.clone_gpmeta(self.gpv_id)
# Clone the replicons as we clone the GP record
update_params = {'version_id': version, 'gpv_id': self.gpv_id}
for rep in session.query(Replicon).filter(
Replicon.gpv_id == old_gpv_id):
logger.debug("Copying replicon {}".format(rep.rpv_id))
rep.copy_and_update(**update_params)
for gpcs in session.query(GenomeProject_Checksum).filter(
GenomeProject_Checksum.gpv_id == old_gpv_id):
logger.debug("Copying GP_Checksum for file {}".format(
gpcs.filename))
gpcs.copy_and_update(**update_params)
except Exception as e:
logger.exception("Exception cloning GenomeProject: " + str(e))
session.rollback()
raise e
def _executable_task_instances_to_queued(self, max_tis: int, session: Session = None) -> List[TI]:
"""
Finds TIs that are ready for execution with respect to pool limits,
dag max_active_tasks, executor state, and priority.
:param max_tis: Maximum number of TIs to queue in this loop.
:type max_tis: int
:return: list[airflow.models.TaskInstance]
"""
executable_tis: List[TI] = []
# Get the pool settings. We get a lock on the pool rows, treating this as a "critical section"
# Throws an exception if lock cannot be obtained, rather than blocking
pools = models.Pool.slots_stats(lock_rows=True, session=session)
# If the pools are full, there is no point doing anything!
# If _somehow_ the pool is overfull, don't let the limit go negative - it breaks SQL
pool_slots_free = max(0, sum(pool['open'] for pool in pools.values()))
if pool_slots_free == 0:
self.log.debug("All pools are full!")
return executable_tis
max_tis = min(max_tis, pool_slots_free)
# Get all task instances associated with scheduled
# DagRuns which are not backfilled, in the given states,
# and the dag is not paused
query = (
session.query(TI)
.join(TI.dag_run)
.options(eagerload(TI.dag_run))
.filter(DR.run_type != DagRunType.BACKFILL_JOB, DR.state != DagRunState.QUEUED)
.join(TI.dag_model)
.filter(not_(DM.is_paused))
.filter(TI.state == State.SCHEDULED)
.options(selectinload('dag_model'))
.order_by(-TI.priority_weight, DR.execution_date)
)
starved_pools = [pool_name for pool_name, stats in pools.items() if stats['open'] <= 0]
if starved_pools:
query = query.filter(not_(TI.pool.in_(starved_pools)))
query = query.limit(max_tis)
task_instances_to_examine: List[TI] = with_row_locks(
query,
of=TI,
session=session,
**skip_locked(session=session),
).all()
# TODO[HA]: This was wrong before anyway, as it only looked at a sub-set of dags, not everything.
# Stats.gauge('scheduler.tasks.pending', len(task_instances_to_examine))
if len(task_instances_to_examine) == 0:
self.log.debug("No tasks to consider for execution.")
return executable_tis
# Put one task instance on each line
task_instance_str = "\n\t".join(repr(x) for x in task_instances_to_examine)
self.log.info("%s tasks up for execution:\n\t%s", len(task_instances_to_examine), task_instance_str)
pool_to_task_instances: DefaultDict[str, List[models.Pool]] = defaultdict(list)
for task_instance in task_instances_to_examine:
pool_to_task_instances[task_instance.pool].append(task_instance)
# dag_id to # of running tasks and (dag_id, task_id) to # of running tasks.
dag_max_active_tasks_map: DefaultDict[str, int]
task_concurrency_map: DefaultDict[Tuple[str, str], int]
dag_max_active_tasks_map, task_concurrency_map = self.__get_concurrency_maps(
states=list(EXECUTION_STATES), session=session
)
num_tasks_in_executor = 0
# Number of tasks that cannot be scheduled because of no open slot in pool
num_starving_tasks_total = 0
# Go through each pool, and queue up a task for execution if there are
# any open slots in the pool.
for pool, task_instances in pool_to_task_instances.items():
pool_name = pool
if pool not in pools:
self.log.warning("Tasks using non-existent pool '%s' will not be scheduled", pool)
continue
open_slots = pools[pool]["open"]
num_ready = len(task_instances)
self.log.info(
"Figuring out tasks to run in Pool(name=%s) with %s open slots "
"and %s task instances ready to be queued",
pool,
open_slots,
num_ready,
)
priority_sorted_task_instances = sorted(
task_instances, key=lambda ti: (-ti.priority_weight, ti.execution_date)
)
num_starving_tasks = 0
for current_index, task_instance in enumerate(priority_sorted_task_instances):
if open_slots <= 0:
self.log.info("Not scheduling since there are %s open slots in pool %s", open_slots, pool)
# Can't schedule any more since there are no more open slots.
num_unhandled = len(priority_sorted_task_instances) - current_index
num_starving_tasks += num_unhandled
num_starving_tasks_total += num_unhandled
break
# Check to make sure that the task max_active_tasks of the DAG hasn't been
# reached.
dag_id = task_instance.dag_id
current_max_active_tasks_per_dag = dag_max_active_tasks_map[dag_id]
max_active_tasks_per_dag_limit = task_instance.dag_model.max_active_tasks
self.log.info(
"DAG %s has %s/%s running and queued tasks",
dag_id,
current_max_active_tasks_per_dag,
max_active_tasks_per_dag_limit,
)
if current_max_active_tasks_per_dag >= max_active_tasks_per_dag_limit:
self.log.info(
"Not executing %s since the number of tasks running or queued "
"from DAG %s is >= to the DAG's max_active_tasks limit of %s",
task_instance,
dag_id,
max_active_tasks_per_dag_limit,
)
continue
task_concurrency_limit: Optional[int] = None
if task_instance.dag_model.has_task_concurrency_limits:
# Many dags don't have a task_concurrency, so where we can avoid loading the full
# serialized DAG the better.
serialized_dag = self.dagbag.get_dag(dag_id, session=session)
if serialized_dag.has_task(task_instance.task_id):
task_concurrency_limit = serialized_dag.get_task(
task_instance.task_id
).max_active_tis_per_dag
if task_concurrency_limit is not None:
current_task_concurrency = task_concurrency_map[
(task_instance.dag_id, task_instance.task_id)
]
if current_task_concurrency >= task_concurrency_limit:
self.log.info(
"Not executing %s since the task concurrency for"
" this task has been reached.",
task_instance,
)
continue
if task_instance.pool_slots > open_slots:
self.log.info(
"Not executing %s since it requires %s slots "
"but there are %s open slots in the pool %s.",
task_instance,
task_instance.pool_slots,
open_slots,
pool,
)
num_starving_tasks += 1
num_starving_tasks_total += 1
# Though we can execute tasks with lower priority if there's enough room
continue
executable_tis.append(task_instance)
open_slots -= task_instance.pool_slots
dag_max_active_tasks_map[dag_id] += 1
task_concurrency_map[(task_instance.dag_id, task_instance.task_id)] += 1
Stats.gauge(f'pool.starving_tasks.{pool_name}', num_starving_tasks)
Stats.gauge('scheduler.tasks.starving', num_starving_tasks_total)
Stats.gauge('scheduler.tasks.running', num_tasks_in_executor)
Stats.gauge('scheduler.tasks.executable', len(executable_tis))
task_instance_str = "\n\t".join(repr(x) for x in executable_tis)
self.log.info("Setting the following tasks to queued state:\n\t%s", task_instance_str)
if len(executable_tis) > 0:
# set TIs to queued state
filter_for_tis = TI.filter_for_tis(executable_tis)
session.query(TI).filter(filter_for_tis).update(
# TODO[ha]: should we use func.now()? How does that work with DB timezone
# on mysql when it's not UTC?
{TI.state: State.QUEUED, TI.queued_dttm: timezone.utcnow(), TI.queued_by_job_id: self.id},
synchronize_session=False,
)
for ti in executable_tis:
make_transient(ti)
return executable_tis
def clone(project_id: int, author_id: int):
""" Clone project """
cloned_project = Project.get(project_id)
# Remove clone from session so we can reinsert it as a new object
db.session.expunge(cloned_project)
make_transient(cloned_project)
# Re-initialise counters and meta-data
cloned_project.total_tasks = 0
cloned_project.tasks_mapped = 0
cloned_project.tasks_validated = 0
cloned_project.tasks_bad_imagery = 0
cloned_project.last_updated = timestamp()
cloned_project.created = timestamp()
cloned_project.author_id = author_id
cloned_project.status = ProjectStatus.DRAFT.value
cloned_project.id = None # Reset ID so we get a new ID when inserted
cloned_project.geometry = None
cloned_project.centroid = None
db.session.add(cloned_project)
db.session.commit()
# Now add the project info, we have to do it in a two stage commit because we need to know the new project id
original_project = Project.get(project_id)
for info in original_project.project_info:
db.session.expunge(info)
make_transient(
info
) # Must remove the object from the session or it will be updated rather than inserted
info.id = None
info.project_id_str = str(cloned_project.id)
cloned_project.project_info.append(info)
# Now add allowed users now we know new project id, if there are any
for user in original_project.allowed_users:
cloned_project.allowed_users.append(user)
# Add other project metadata
cloned_project.priority = original_project.priority
cloned_project.default_locale = original_project.default_locale
cloned_project.mapper_level = original_project.mapper_level
cloned_project.mapping_permission = original_project.mapping_permission
cloned_project.validation_permission = original_project.validation_permission
cloned_project.enforce_random_task_selection = (
original_project.enforce_random_task_selection
)
cloned_project.private = original_project.private
cloned_project.entities_to_map = original_project.entities_to_map
cloned_project.due_date = original_project.due_date
cloned_project.imagery = original_project.imagery
cloned_project.josm_preset = original_project.josm_preset
cloned_project.license_id = original_project.license_id
cloned_project.mapping_types = original_project.mapping_types
# We try to remove the changeset comment referencing the old project. This
# assumes the default changeset comment has not changed between the old
# project and the cloned. This is a best effort basis.
default_comment = current_app.config["DEFAULT_CHANGESET_COMMENT"]
changeset_comments = []
if original_project.changeset_comment is not None:
changeset_comments = original_project.changeset_comment.split(" ")
if f"{default_comment}-{original_project.id}" in changeset_comments:
changeset_comments.remove(f"{default_comment}-{original_project.id}")
cloned_project.changeset_comment = " ".join(changeset_comments)
db.session.add(cloned_project)
db.session.commit()
return cloned_project
def SPStep1(self):
logging.info("Input Param = %s" % (self.param.__dict__, ))
# self.param.ProjectId="2"
Prj = database.Projects.query.filter_by(
projid=self.param.ProjectId).first()
# self.param.IntraStep=0
if getattr(self.param, 'IntraStep', 0) == 0:
self.param.IntraStep = 1
db.session.expunge(Prj)
NewPrj = Prj
Prj = copy.copy(
NewPrj) # Si on fait une copy on arrive plus à insérer.
make_transient(NewPrj)
NewPrj.title = self.param.subsetprojecttitle
NewPrj.projid = None
NewPrj.visible = False
db.session.add(NewPrj)
db.session.commit()
pp = database.ProjectsPriv()
pp.member = self.task.owner_id
pp.privilege = "Manage"
NewPrj.projmembers.append(pp)
db.session.commit()
self.param.subsetproject = NewPrj.projid
self.UpdateProgress(
5, "Subset Project %d Created : %s" %
(NewPrj.projid, NewPrj.title))
if self.param.IntraStep == 1:
vaultroot = Path("../../vault")
sqlparam = {'projid': self.param.ProjectId}
sqlwhere = ""
if self.param.extraprojects:
sqlparam['projid'] += "," + self.param.extraprojects
sqlparam['ranklimit'] = self.param.valeur
if self.param.valtype == 'V': rankfunction = 'rank'
elif self.param.valtype == 'P': rankfunction = '100*percent_rank'
else: rankfunction = 'FunctionError'
# if self.param.samplelist:
# sqlwhere+=" and s.orig_id in (%s) "%(",".join(["'%s'"%x for x in self.param.samplelist.split(",")]))
# sqlwhere+=" and (o.classif_qual in (%s) "%(",".join(["'%s'"%x for x in self.param.what.split(",")]))
# if self.param.what.find('N')>=0:
# sqlwhere+=" or o.classif_qual is null "
# sqlwhere+=")"
sqlwhere += sharedfilter.GetSQLFilter(self.param.filtres, sqlparam,
str(self.task.owner_id))
logging.info("SQLParam=%s", sqlparam)
sql = """select objid from (
SELECT """ + rankfunction + """() OVER (partition by classif_id order by random() )rang,o.objid
from objects o left join samples s on o.sampleid=s.sampleid
where o.projid in ( %(projid)s ) """ + sqlwhere + """ ) sr
where rang<=%(ranklimit)s """
logging.info("SQL=%s %s", sql, sqlparam)
# for obj in db.session.query(database.Objects).from_statement( text(sql) ).all():
LstObjects = GetAll(sql, sqlparam)
logging.info("matched %s objects", len(LstObjects))
if len(LstObjects) == 0:
self.task.taskstate = "Error"
self.UpdateProgress(
10, "No object to include in the subset project")
NbrObjects = 0
for objid in LstObjects:
obj = db.session.query(
database.Objects).filter_by(objid=objid[0]).first()
objf = db.session.query(
database.ObjectsFields).filter_by(objfid=objid[0]).first()
objcnn = db.session.query(
database.Objects_cnn_features).filter_by(
objcnnid=objid[0]).first()
NbrObjects += 1
oldobjid = obj.objid
if self.param.withimg == 'Y':
for img in obj.images:
db.session.expunge(img)
make_transient(img)
self.pgcur.execute("select nextval('seq_images')")
img.imgid = self.pgcur.fetchone()[0]
# print("New Image id=",img.imgid)
SrcImg = img.file_name
SrcImgMini = img.thumb_file_name
VaultFolder = "%04d" % (img.imgid // 10000)
#creation du repertoire contenant les images si necessaire
CreateDirConcurrentlyIfNeeded(
vaultroot.joinpath(VaultFolder))
img.file_name = "%s/%04d%s" % (VaultFolder,
img.imgid % 10000,
Path(SrcImg).suffix)
shutil.copyfile(
vaultroot.joinpath(SrcImg).as_posix(),
vaultroot.joinpath(img.file_name).as_posix())
if SrcImgMini is not None:
img.thumb_file_name = "%s/%04d_mini%s" % (
VaultFolder, img.imgid % 10000,
Path(SrcImgMini).suffix)
shutil.copyfile(
vaultroot.joinpath(SrcImgMini).as_posix(),
vaultroot.joinpath(
img.thumb_file_name).as_posix())
db.session.expunge(obj)
make_transient(obj)
obj.objid = None
obj.img0id = None
obj.projid = self.param.subsetproject
obj.sampleid = self.GetSampleID(obj.sampleid)
obj.processid = self.GetProcessID(obj.processid)
obj.acquisid = self.GetAcquisID(obj.acquisid)
db.session.add(obj)
db.session.commit()
dummy = objf.n01 #permet de forcer l'etat de objf sinon perd ses données sur les instruction suivantes.
db.session.expunge(objf)
make_transient(objf)
objf.objfid = obj.objid
db.session.add(objf)
if objcnn:
dummy = objcnn.cnn01 #permet de forcer l'etat de objcnn sinon perd ses données sur les instruction suivantes.
db.session.expunge(objcnn)
make_transient(objcnn)
objcnn.objcnnid = obj.objid
db.session.add(objcnn)
db.session.commit()
if NbrObjects % 20 == 0:
self.UpdateProgress(5 + 95 * NbrObjects / len(LstObjects),
"Subset creation in progress")
# print (oldobjid,obj.objid)
# Recalcule les valeurs de Img0
self.pgcur.execute("""update obj_head o
set imgcount=(select count(*) from images where objid=o.objid)
,img0id=(select imgid from images where objid=o.objid order by imgrank asc limit 1 )
where projid=""" +
str(self.param.subsetproject))
self.pgcur.connection.commit()
import appli.project.main
appli.project.main.RecalcProjectTaxoStat(self.param.subsetproject)
appli.project.main.UpdateProjectStat(self.param.subsetproject)
self.task.taskstate = "Done"
self.UpdateProgress(100, "Subset created successfully")
def duplicate_instance(item):
db.session.expunge(item)
make_transient(item)
item.id = None
db.session.add(item)
return item
def on_delete(self, req, resp, *args, **kwargs):
"""
Delete a single item.
"""
if 'DELETE' not in getattr(self, 'methods',
['GET', 'PUT', 'PATCH', 'DELETE']):
raise falcon.errors.HTTPMethodNotAllowed(
getattr(self, 'methods', ['GET', 'PUT', 'PATCH', 'DELETE']))
with session_scope(self.db_engine,
sessionmaker_=self.sessionmaker,
**self.sessionmaker_kwargs) as db_session:
resources = self.apply_arg_filter(req, resp,
db_session.query(self.model),
kwargs)
try:
resource = resources.one()
except sqlalchemy.orm.exc.NoResultFound:
raise falcon.errors.HTTPNotFound()
except sqlalchemy.orm.exc.MultipleResultsFound:
self.logger.error(
'Programming error: multiple results found for patch of model {0}'
.format(self.model))
raise falcon.errors.HTTPInternalServerError(
'Internal Server Error',
'An internal server error occurred')
resources = self.delete_precondition(
req, resp, self.filter_by_params(resources, req.params), *args,
**kwargs)
try:
resource = resources.one()
except sqlalchemy.orm.exc.NoResultFound:
raise falcon.errors.HTTPConflict(
'Conflict', 'Resource found but conditions violated')
except sqlalchemy.orm.exc.MultipleResultsFound:
self.logger.error(
'Programming error: multiple results found for delete of model {0}'
.format(self.model))
raise falcon.errors.HTTPInternalServerError(
'Internal Server Error',
'An internal server error occurred')
before_delete = getattr(self, 'before_delete', None)
if before_delete is not None:
self.before_delete(req, resp, db_session, resource, *args,
**kwargs)
try:
mark_deleted = getattr(self, 'mark_deleted', None)
if mark_deleted is not None:
mark_deleted(req, resp, resource, *args, **kwargs)
db_session.add(resource)
else:
make_transient(resource)
resources.delete()
db_session.commit()
except sqlalchemy.exc.IntegrityError as err:
# As far we I know, this should only be caused by foreign key constraint being violated
db_session.rollback()
raise falcon.errors.HTTPConflict(
'Conflict', 'Other content links to this')
except sqlalchemy.orm.exc.StaleDataError as err:
# Version field in the model was not as expected
db_session.rollback()
raise falcon.errors.HTTPConflict(
'Conflict', 'Resource found but conditions violated')
except sqlalchemy.exc.ProgrammingError as err:
db_session.rollback()
if self._is_foreign_key_violation(err):
raise falcon.errors.HTTPConflict(
'Conflict', 'Other content links to this')
else:
raise
resp.status = falcon.HTTP_OK
req.context['result'] = {}
after_delete = getattr(self, 'after_delete', None)
if after_delete is not None:
after_delete(req, resp, resource, *args, **kwargs)
def test_predictor_retrieve():
with testing.postgresql.Postgresql() as postgresql:
db_engine = create_engine(postgresql.url())
ensure_db(db_engine)
project_path = 'econ-dev/inspections'
model_storage_engine = InMemoryModelStorageEngine(project_path)
_, model_id = \
fake_trained_model(project_path, model_storage_engine, db_engine)
predictor = Predictor(project_path, model_storage_engine, db_engine, replace=False)
dayone = datetime.date(2011, 1, 1).isoformat()
daytwo = datetime.date(2011, 1, 2).isoformat()
# create prediction set
matrix_data = {
'entity_id': [1, 2, 1, 2],
'as_of_date': [dayone, dayone, daytwo, daytwo],
'feature_one': [3, 4, 5, 6],
'feature_two': [5, 6, 7, 8],
'label': [7, 8, 8, 7]
}
matrix = pandas.DataFrame.from_dict(matrix_data)\
.set_index(['entity_id', 'as_of_date'])
metadata = {
'label_name': 'label',
'end_time': AS_OF_DATE,
'label_window': '3month',
'metta-uuid': '1234',
}
matrix_store = InMemoryMatrixStore(matrix, metadata)
predict_proba = predictor.predict(model_id, matrix_store, misc_db_parameters=dict())
# When run again, the predictions retrieved from the database
# should match.
#
# Some trickiness here. Let's explain:
#
# If we are not careful, retrieving predictions from the database and
# presenting them as a numpy array can result in a bad ordering,
# since the given matrix may not be 'ordered' by some criteria
# that can be easily represented by an ORDER BY clause.
#
# It will sometimes work, because without ORDER BY you will get
# it back in the table's physical order, which unless something has
# happened to the table will be the order you inserted it,
# which could very well be the order in the matrix.
# So it's not a bug that would necessarily immediately show itself,
# but when it does go wrong your scores will be garbage.
#
# So we simulate a table order mutation that can happen over time:
# Remove the first row and put it at the end.
# If the Predictor doesn't explicitly reorder the results, this will fail
session = sessionmaker(bind=db_engine)()
obj = session.query(Prediction).first()
session.delete(obj)
session.commit()
make_transient(obj)
session = sessionmaker(bind=db_engine)()
session.add(obj)
session.commit()
predictor.load_model = Mock()
new_predict_proba = predictor.predict(model_id, matrix_store, misc_db_parameters=dict())
assert_array_equal(new_predict_proba, predict_proba)
assert not predictor.load_model.called
def import_files(files: List[CSVFile]):
new_ids = {str(c.id): uuid4() for c in files}
todo: List[CSVFile] = files.copy()
imported: List[CSVFile] = []
while todo:
csv: CSVFile = todo.pop(0)
new_id = new_ids.get(str(csv.id))
ori_id = csv.id
table = csv.table
# remove from session
for sub in csv.columns + csv.rows + csv.group_levels:
db.session.expunge(sub)
make_transient(sub)
sub.csv_file_id = new_id
db.session.expunge(csv)
make_transient(csv)
# update releated things
csv.id = new_id
csv.created = datetime.utcnow()
csv.meta = csv.meta.copy()
csv.sample_group = None
# extra option to mimic the _read_csv logic
csv.save_table(table, index=False, header=False)
if 'merged' in csv.meta:
# ensure we also merge the others
merged_ids: List[str] = csv.meta['merged']
new_merged_ids: List[str] = []
for file_id in merged_ids:
if file_id not in new_ids:
extra: CSVFile = CSVFile.query.get(file_id)
if extra:
extra_new_id = uuid4()
new_ids[extra.id] = extra_new_id
todo.append(extra)
new_merged_ids.append(str(extra_new_id))
else:
new_merged_ids.append(str(new_ids[file_id]))
csv.meta['merged'] = new_merged_ids
db.session.add(csv)
for sub in csv.columns + csv.rows + csv.group_levels:
db.session.add(sub)
imported.append(csv)
v: ValidatedMetaboliteTable = ValidatedMetaboliteTable.query \
.filter_by(csv_file_id=ori_id).first()
if not v:
continue
db.session.expunge(v)
make_transient(v)
v.id = uuid4()
v.csv_file_id = csv.id
v.created = csv.created
v.meta = csv.meta.copy()
db.session.add(v)
return imported
def test_import_dashboard_2_slices(self):
schema = get_example_default_schema()
e_slc = self.create_slice("e_slc",
id=10007,
table_name="energy_usage",
schema=schema)
b_slc = self.create_slice("b_slc",
id=10008,
table_name="birth_names",
schema=schema)
dash_with_2_slices = self.create_dashboard("dash_with_2_slices",
slcs=[e_slc, b_slc],
id=10003)
dash_with_2_slices.json_metadata = json.dumps({
"remote_id": 10003,
"expanded_slices": {
"{}".format(e_slc.id): True,
"{}".format(b_slc.id): False,
},
# mocked filter_scope metadata
"filter_scopes": {
str(e_slc.id): {
"region": {
"scope": ["ROOT_ID"],
"immune": [b_slc.id]
}
}
},
})
imported_dash_id = import_dashboard(dash_with_2_slices,
import_time=1991)
imported_dash = self.get_dash(imported_dash_id)
expected_dash = self.create_dashboard("dash_with_2_slices",
slcs=[e_slc, b_slc],
id=10003)
make_transient(expected_dash)
self.assert_dash_equals(imported_dash,
expected_dash,
check_position=False,
check_slugs=False)
i_e_slc = self.get_slice_by_name("e_slc")
i_b_slc = self.get_slice_by_name("b_slc")
expected_json_metadata = {
"remote_id": 10003,
"import_time": 1991,
"filter_scopes": {
str(i_e_slc.id): {
"region": {
"scope": ["ROOT_ID"],
"immune": [i_b_slc.id]
}
}
},
"expanded_slices": {
"{}".format(i_e_slc.id): True,
"{}".format(i_b_slc.id): False,
},
}
self.assertEqual(expected_json_metadata,
json.loads(imported_dash.json_metadata))
def test_predictor_retrieve(predict_setup_args):
"""Test the predictions retrieved from the database match the output from predict_proba"""
(project_storage, db_engine, model_id) = predict_setup_args
predictor = Predictor(project_storage.model_storage_engine(),
db_engine,
'worst',
replace=False)
# create prediction set
matrix_store = get_matrix_store(project_storage)
predict_proba = predictor.predict(
model_id,
matrix_store,
misc_db_parameters=dict(),
train_matrix_columns=matrix_store.columns())
# When run again, the predictions retrieved from the database
# should match.
#
# Some trickiness here. Let's explain:
#
# If we are not careful, retrieving predictions from the database and
# presenting them as a numpy array can result in a bad ordering,
# since the given matrix may not be 'ordered' by some criteria
# that can be easily represented by an ORDER BY clause.
#
# It will sometimes work, because without ORDER BY you will get
# it back in the table's physical order, which unless something has
# happened to the table will be the order you inserted it,
# which could very well be the order in the matrix.
# So it's not a bug that would necessarily immediately show itself,
# but when it does go wrong your scores will be garbage.
#
# So we simulate a table order mutation that can happen over time:
# Remove the first row and put it at the end.
# If the Predictor doesn't explicitly reorder the results, this will fail
# Only running on TestPrediction because TrainPrediction behaves the exact same way
try:
reorder_session = sessionmaker(bind=db_engine)()
obj = reorder_session.query(TestPrediction).first()
reorder_session.delete(obj)
reorder_session.commit()
finally:
reorder_session.close()
make_transient(obj)
try:
reorder_session = sessionmaker(bind=db_engine)()
reorder_session.add(obj)
reorder_session.commit()
finally:
reorder_session.close()
predictor.load_model = Mock()
new_predict_proba = predictor.predict(
model_id,
matrix_store,
misc_db_parameters=dict(),
train_matrix_columns=matrix_store.columns())
assert_array_almost_equal(new_predict_proba, predict_proba, decimal=5)
assert not predictor.load_model.called
def _do_update(self, session, obj, existing_obj):
"""Updates the associated ParticipantSummary, and extracts HPO ID from the provider link
or set pairing at another level (site/organization/awardee) with parent/child enforcement."""
obj.lastModified = clock.CLOCK.now()
obj.signUpTime = existing_obj.signUpTime
obj.biobankId = existing_obj.biobankId
obj.withdrawalTime = existing_obj.withdrawalTime
obj.suspensionTime = existing_obj.suspensionTime
need_new_summary = False
if obj.withdrawalStatus != existing_obj.withdrawalStatus:
obj.withdrawalTime = (obj.lastModified if obj.withdrawalStatus
== WithdrawalStatus.NO_USE else None)
need_new_summary = True
if obj.suspensionStatus != existing_obj.suspensionStatus:
obj.suspensionTime = (obj.lastModified if obj.suspensionStatus
== SuspensionStatus.NO_CONTACT else None)
need_new_summary = True
update_pairing = True
if obj.siteId is None and obj.organizationId is None and obj.hpoId is None and \
obj.providerLink == 'null':
# Prevent unpairing if /PUT is sent with no pairing levels.
update_pairing = False
if update_pairing is True:
has_id = False
if obj.organizationId or obj.siteId or (obj.hpoId >= 0):
has_id = True
provider_link_unchanged = True
if obj.providerLink is not None:
if existing_obj.providerLink:
provider_link_unchanged = json.loads(obj.providerLink) == \
json.loads(existing_obj.providerLink)
else:
provider_link_unchanged = False
null_provider_link = obj.providerLink == 'null'
# site,org,or awardee is sent in request: Get relationships and try to set provider link.
if has_id and (provider_link_unchanged or null_provider_link):
site, organization, awardee = self.get_pairing_level(obj)
obj.organizationId = organization
obj.siteId = site
obj.hpoId = awardee
if awardee is not None and (obj.hpoId != existing_obj.hpoId):
# get provider link for hpo_id (awardee)
obj.providerLink = make_primary_provider_link_for_id(
awardee)
need_new_summary = True
else: # providerLink has changed
# If the provider link changes, update the HPO ID on the participant and its summary.
if obj.hpoId is None:
obj.hpoId = existing_obj.hpoId
new_hpo_id = self._get_hpo_id(obj)
if new_hpo_id != existing_obj.hpoId:
obj.hpoId = new_hpo_id
obj.siteId = None
obj.organizationId = None
need_new_summary = True
# No pairing updates sent, keep existing values.
if update_pairing == False:
obj.siteId = existing_obj.siteId
obj.organizationId = existing_obj.organizationId
obj.hpoId = existing_obj.hpoId
obj.providerLink = existing_obj.providerLink
if need_new_summary and existing_obj.participantSummary:
# Copy the existing participant summary, and mutate the fields that
# come from participant.
summary = existing_obj.participantSummary
summary.hpoId = obj.hpoId
summary.organizationId = obj.organizationId
summary.siteId = obj.siteId
summary.withdrawalStatus = obj.withdrawalStatus
summary.withdrawalReason = obj.withdrawalReason
summary.withdrawalReasonJustification = obj.withdrawalReasonJustification
summary.withdrawalTime = obj.withdrawalTime
summary.suspensionStatus = obj.suspensionStatus
summary.suspensionTime = obj.suspensionTime
summary.lastModified = clock.CLOCK.now()
make_transient(summary)
make_transient(obj)
obj.participantSummary = summary
self._update_history(session, obj, existing_obj)
super(ParticipantDao, self)._do_update(session, obj, existing_obj)
def import_datasource( # pylint: disable=too-many-arguments
session: Session,
i_datasource: Model,
lookup_database: Callable[[Model], Optional[Model]],
lookup_datasource: Callable[[Model], Optional[Model]],
import_time: Optional[int] = None,
database_id: Optional[int] = None,
) -> int:
"""Imports the datasource from the object to the database.
Metrics and columns and datasource will be overrided if exists.
This function can be used to import/export datasources between multiple
superset instances. Audit metadata isn't copies over.
"""
make_transient(i_datasource)
logger.info("Started import of the datasource: %s", i_datasource.to_json())
i_datasource.id = None
i_datasource.database_id = (database_id if database_id else getattr(
lookup_database(i_datasource), "id", None))
i_datasource.alter_params(import_time=import_time)
# override the datasource
datasource = lookup_datasource(i_datasource)
if datasource:
datasource.override(i_datasource)
session.flush()
else:
datasource = i_datasource.copy()
session.add(datasource)
session.flush()
for metric in i_datasource.metrics:
new_m = metric.copy()
new_m.table_id = datasource.id
logger.info(
"Importing metric %s from the datasource: %s",
new_m.to_json(),
i_datasource.full_name,
)
imported_m = import_metric(session, new_m)
if imported_m.metric_name not in [
m.metric_name for m in datasource.metrics
]:
datasource.metrics.append(imported_m)
for column in i_datasource.columns:
new_c = column.copy()
new_c.table_id = datasource.id
logger.info(
"Importing column %s from the datasource: %s",
new_c.to_json(),
i_datasource.full_name,
)
imported_c = import_column(session, new_c)
if imported_c.column_name not in [
c.column_name for c in datasource.columns
]:
datasource.columns.append(imported_c)
session.flush()
return datasource.id
def clone_model(db: Session, model, **kwargs):
db.expunge(model)
make_transient(model)
for (field, value) in kwargs.items():
setattr(model, field, value)
return model
def get_plain(cls, name):
with new_session() as session:
project = session.query(Project).filter_by(name=name).first()
make_transient(project)
return project
def test_dup_db_entry_refresh(setup_temp_appliance_provider,
temp_appliance_preconfig, provider):
"""
Polarion:
assignee: juwatts
caseimportance: critical
casecomponent: Containers
initialEstimate: 1/6h
Bugzilla:
1732442
1749060
"""
appliance = temp_appliance_preconfig
image_table = appliance.db.client['container_groups']
image_query = appliance.db.client.session.query(image_table)
all_db_entries = image_query.all()
if not all_db_entries:
pytest.fail("No Entries in the containter_groups DB table")
# Grab the first entry in the table
db_entry = all_db_entries[0]
copied_db_entry = deepcopy(db_entry)
# Remove the object from the session
appliance.db.client.session.expunge(db_entry)
make_transient(db_entry)
# ID is the primary key, set it to something high
db_entry_last = all_db_entries[-1]
copied_db_entry.id = db_entry_last.id + 500
try:
with appliance.db.client.transaction:
appliance.db.client.session.add(copied_db_entry)
except IntegrityError as ex:
pytest.fail('Exception while adding DB entry. {}'.format(ex))
new_db_entry = image_query.filter(
image_table.id == copied_db_entry.id).all()
# Should only be one entry
assert len(new_db_entry) == 1
for column, value in vars(new_db_entry[0]).items():
# _sa_instance_state is an sqlalchemy InstanceState key object
if column == "_sa_instance_state":
continue
elif column == "id":
assert value != getattr(db_entry, column)
else:
# Verify the entries in the DB are the same
assert value == getattr(db_entry, column)
with LogValidator(
'/var/www/miq/vmdb/log/evm.log',
failure_patterns=['.*nil:NilClass.*'],
).waiting(timeout=600):
provider.refresh_provider_relationships()
wait_for(provider.is_refreshed,
func_kwargs=dict(refresh_delta=10),
timeout=600)
def test_get_reusable_module_use_latest_build(self, cfg, allow_ocbm):
"""
Test that the `get_reusable_module` tries to reuse the latest module in case when
multiple modules can be reused allow_only_compatible_base_modules is True.
"""
cfg.return_value = allow_ocbm
# Set "fedora" virtual stream to platform:f28.
platform_f28 = db_session.query(models.ModuleBuild).filter_by(name="platform").one()
mmd = platform_f28.mmd()
xmd = mmd.get_xmd()
xmd["mbs"]["virtual_streams"] = ["fedora"]
mmd.set_xmd(xmd)
platform_f28.modulemd = mmd_to_str(mmd)
platform_f28.update_virtual_streams(db_session, ["fedora"])
# Create platform:f29 with "fedora" virtual stream.
mmd = load_mmd(read_staged_data("platform"))
mmd = mmd.copy("platform", "f29")
xmd = mmd.get_xmd()
xmd["mbs"]["virtual_streams"] = ["fedora"]
mmd.set_xmd(xmd)
platform_f29 = import_mmd(db_session, mmd)[0]
# Create another copy of `testmodule:master` which should be reused, because its
# stream version will be higher than the previous one. Also set its buildrequires
# to platform:f29.
latest_module = db_session.query(models.ModuleBuild).filter_by(
name="testmodule", state=models.BUILD_STATES["ready"]).one()
# This is used to clone the ModuleBuild SQLAlchemy object without recreating it from
# scratch.
db_session.expunge(latest_module)
make_transient(latest_module)
# Change the platform:f28 buildrequirement to platform:f29 and recompute the build_context.
mmd = latest_module.mmd()
xmd = mmd.get_xmd()
xmd["mbs"]["buildrequires"]["platform"]["stream"] = "f29"
mmd.set_xmd(xmd)
latest_module.modulemd = mmd_to_str(mmd)
contexts = models.ModuleBuild.contexts_from_mmd(
latest_module.modulemd
)
latest_module.build_context = contexts.build_context
latest_module.context = contexts.context
latest_module.buildrequires = [platform_f29]
# Set the `id` to None, so new one is generated by SQLAlchemy.
latest_module.id = None
db_session.add(latest_module)
db_session.commit()
module = db_session.query(models.ModuleBuild)\
.filter_by(name="testmodule")\
.filter_by(state=models.BUILD_STATES["build"])\
.one()
db_session.commit()
reusable_module = get_reusable_module(module)
if allow_ocbm:
assert reusable_module.id == latest_module.id
else:
# There are two testmodules in ready state, the first one with
# lower id is what we want.
first_module = db_session.query(models.ModuleBuild).filter_by(
name="testmodule", state=models.BUILD_STATES["ready"]
).order_by(models.ModuleBuild.id).first()
assert reusable_module.id == first_module.id
