def run(cls, env):
return # disabled by request of Sergey
db_session = DBSession()
transaction.manager.begin()
LogEntry.info('InternalUpdateReactor started!', component=COMP_ID, group=InternalUpdateReactor.identity, append_dt=datetime.now())
fs_resources = db_session.query(FoclStruct).all()
for fs in fs_resources:
# get const obj
try:
const_obj = db_session.query(ConstructObject).filter(ConstructObject.resource_id == fs.id).one()
except:
LogEntry.info('Error on update const obj with id = ' + str(fs.id),
component=COMP_ID, group=InternalUpdateReactor.identity, append_dt=datetime.now())
continue
# update from project vector layers
const_obj.fosc_plan = cls.get_feat_count_for_layer(fs, 'fosc')
const_obj.cross_plan = cls.get_feat_count_for_layer(fs, 'optical_cross')
const_obj.spec_trans_plan = cls.get_feat_count_for_layer(fs, 'special_transition')
db_session.flush()
LogEntry.info('InternalUpdateReactor finished!', component=COMP_ID, group=InternalUpdateReactor.identity, append_dt=datetime.now())
transaction.manager.commit()
def load_domain_dicts(cls, force=False, federal=True, region=True, district=True):
print 'Loading domain dicts...'
from ..compulink_admin.model import Region, District
from csv import DictReader
if (region and not federal) or (district and not region):
print('Not consist params!')
return
db_session = DBSession()
db_session.autoflush = False
with transaction.manager:
if ((db_session.query(FederalDistrict).count() > 0 and federal) or
(db_session.query(Region).count() > 0 and region) or
(db_session.query(District).count() > 0 and district)) and not force:
print ' Domain dictionary already existings! Returning...'
return
with open(path.join(BASE_PATH, 'federal_districts.csv')) as fed_csv, \
open(path.join(BASE_PATH, 'regions.csv')) as reg_csv, \
open(path.join(BASE_PATH, 'districts.csv')) as dist_csv:
fed_reader = DictReader(fed_csv)
reg_reader = DictReader(reg_csv)
dist_reader = DictReader(dist_csv)
feds = {}
if federal:
for fed_row in fed_reader:
federal_dist = FederalDistrict()
federal_dist.name = fed_row['name']
federal_dist.short_name = fed_row['short_name']
federal_dist.persist()
feds[fed_row['id']] = federal_dist
regs = {}
if region:
for reg_row in reg_reader:
region = Region()
region.name = reg_row['name']
region.short_name = reg_row['short_name']
region.region_code = reg_row['region_code']
region.persist()
regs[reg_row['id']] = region
if district:
for dist_row in dist_reader:
district = District()
district.name = dist_row['name']
district.short_name = dist_row['short_name']
district.region = regs[dist_row['region_id']]
district.persist()
db_session.flush()
def run(cls, env):
ngw_session = NgwSession()
ms_session = MsSqlSession()
transaction.manager.begin()
LogEntry.info('ExternalUpdateReactor started!', component=COMP_ID, group=ExternalUpdateReactor.identity, append_dt=datetime.now())
# get mssql conn info
enabled_sett = env.compulink_mssql_bridge.settings.get('enable', 'false').lower()
mssql_enable = enabled_sett in ('true', 'yes', '1')
if not mssql_enable:
LogEntry.info('MSSQL disabled in config! Returning...', component=COMP_ID, group=ExternalUpdateReactor.identity, append_dt=datetime.now())
return
# get all external ids
fs_external_ids = ngw_session.query(ConstructObject.external_id).filter(ConstructObject.external_id != None, ConstructObject.external_id != '').all()
fs_external_ids = [r for (r,) in fs_external_ids]
# get info from mssql
CompulinkMssqlBridgeComponent.configure_db_conn(env.compulink_mssql_bridge.settings.get('conn_str', 'no'))
ms_rows = ms_session.query(MssqConstObject)\
.filter(MssqConstObject.ObjectID.in_(fs_external_ids))\
.options(joinedload_all(MssqConstObject.Work3), joinedload_all(MssqConstObject.Work4)).all()
for ms_row in ms_rows:
# get const obj
try:
const_obj = ngw_session.query(ConstructObject).filter(ConstructObject.external_id == str(ms_row.ObjectID)).one()
except:
LogEntry.info('Error on update const obj with ext id = ' + str(ms_row.ObjectID),
component=COMP_ID, group=ExternalUpdateReactor.identity, append_dt=datetime.now())
continue
# update from mssql
const_obj.cabling_plan = ms_row.PreliminaryLineLength #new requ TODO: check!
const_obj.access_point_plan = ms_row.AccessPointAmount #new requ
const_obj.subcontr_name = ms_row.Work3.SubContractor.ContractorName if ms_row.Work3 and ms_row.Work3.SubContractor else None
const_obj.start_build_date = ms_row.Work3.AgreementStartDateWork if ms_row.Work3 else None
const_obj.end_build_date = ms_row.Work3.AgreementFinishDateWork if ms_row.Work3 else None
const_obj.start_deliver_date = ms_row.Work4.AgreementStartDateWork if ms_row.Work4 else None
const_obj.end_deliver_date = ms_row.Work4.AgreementFinishDateWork if ms_row.Work4 else None
ngw_session.flush()
LogEntry.info('ExternalUpdateReactor finished!', component=COMP_ID, group=ExternalUpdateReactor.identity, append_dt=datetime.now())
transaction.manager.commit()
def copy_existing_real_features(args):
db_session = DBSession()
transaction.manager.begin()
fs_resources = db_session.query(FoclStruct).all()
for focl_struct in fs_resources:
layers = focl_struct.children
real_layer = None
actual_layer = None
for real_layer_name in FOCL_REAL_LAYER_STRUCT:
# get real layer and actual layer
for lyr in layers:
if lyr.keyname:
lyr_name = '_'.join(lyr.keyname.rsplit('_')[0:-1])
else:
continue
if real_layer_name == lyr_name:
real_layer = lyr
if 'actual_' + real_layer_name == lyr_name:
actual_layer = lyr
if not real_layer or not actual_layer:
print 'Ops!!!!!!!!!!!!!!!! Needed layers not found!'
return
try:
# clean actual layer
actual_layer.feature_delete_all()
# copy data from real layer
query = real_layer.feature_query()
query.geom()
for feat in query():
feat.fields['change_author'] = u'Мобильное приложение'
feat.fields['change_date'] = feat.fields['built_date']
actual_layer.feature_create(feat)
print "Layers %s was updated!" % actual_layer.keyname
except Exception, ex:
print "Error on update %s: %s" % (actual_layer.keyname, ex.message)
db_session.flush()
def run(cls, env):
db_session = DBSession()
transaction.manager.begin()
cls.log_info('ConstructSpecTransitionLineReactor started!')
fs_resources = db_session.query(FoclStruct).all()
for fs in fs_resources:
cls.smart_construct_line(fs)
db_session.flush()
cls.log_info('ConstructSpecTransitionLineReactor finished!')
transaction.manager.commit()
def drop_vector_layer_table(table_uid, make_transaction=False):
db_session = DBSession()
#start transaction
if make_transaction:
transaction.manager.begin()
engine = db_session.get_bind()
engine.execute('DROP TABLE "vector_layer"."layer_%s"' % table_uid)
#close transaction
if make_transaction:
transaction.manager.commit()
else:
db_session.flush()
def load_rt_domain_dicts(cls, force=False):
print 'Loading RT domain dicts...'
from ..compulink_admin.model import Region
from ..compulink_reporting.model import RtMacroDivision, RtBranch, RtBranchRegion
from csv import DictReader
db_session = DBSession()
db_session.autoflush = False
if (db_session.query(RtMacroDivision).count() > 0 or
db_session.query(RtBranch).count() > 0 or
db_session.query(RtBranchRegion).count() > 0
) and not force:
print ' RT Domain dictionary already existings! Returning...'
return
with open(path.join(BASE_PATH, 'rt_macro_division.csv')) as macro_csv, \
open(path.join(BASE_PATH, 'rt_branch.csv')) as branch_csv, \
open(path.join(BASE_PATH, 'rt_branch_region.csv')) as branch_region_csv:
macro_reader = DictReader(macro_csv)
branch_reader = DictReader(branch_csv)
branch_region_reader = DictReader(branch_region_csv)
macros = {}
branches = {}
for macro_row in macro_reader:
macro = RtMacroDivision()
macro.name = macro_row['name']
macro.persist()
macros[macro_row['id']] = macro
for branch_row in branch_reader:
branch = RtBranch()
branch.name = branch_row['name']
branch.rt_macro_division = macros[branch_row['macro_division_id']]
branch.persist()
branches[branch_row['id']] = branch
for br_reg_row in branch_region_reader:
branch_reg = RtBranchRegion()
branch_reg.rt_branch = branches[br_reg_row['rt_branch_id']]
branch_reg.region = db_session.query(Region).filter(Region.region_code == br_reg_row['region_code']).one()
branch_reg.persist()
db_session.flush()
def run(cls, env):
db_session = DBSession()
transaction.manager.begin()
cls.log_info('ConstructFoclLineReactor started! (%s)' % CONNECTOR_NAME)
fs_resources = db_session.query(FoclStruct).all()
for fs in fs_resources:
try:
cls.smart_construct_line(fs)
db_session.flush()
except Exception as ex:
print 'Error on construct line %s: %s' % (fs.id, ex.message)
cls.log_error('Error on construct line %s: %s' % (fs.id, ex.message))
cls.log_info('ConstructFoclLineReactor finished!')
transaction.manager.commit()
def remove_video_entry(video_id):
db_session = DBSession()
transaction.manager.begin()
task = VideoProduceTask.filter(VideoProduceTask.id == video_id).first()
if task:
fileobjs = FileObj.filter(FileObj.id == task.fileobj_id, FileObj.component == COMP_ID)
for fileobj in fileobjs:
fn = env.file_storage.filename(fileobj)
db_session.delete(fileobj)
try:
remove(fn)
except:
pass
db_session.delete(task)
db_session.flush()
transaction.manager.commit()
return success_response()
def fill_guid_field(args):
db_session = DBSession()
transaction.manager.begin()
resources = db_session.query(VectorLayer).options(joinedload_all('fields')).filter(VectorLayer.keyname.like('real_%')).all()
for vec_layer in resources:
try:
query = vec_layer.feature_query()
query.geom()
for feat in query():
if not feat.fields['feat_guid']:
feat.fields['feat_guid'] = str(uuid.uuid4().hex)
vec_layer.feature_put(feat)
print "GUIDs of %s was updated!" % vec_layer.keyname
except Exception, ex:
print "Error on update GUIDs %s: %s" % (vec_layer.keyname, ex.message)
db_session.flush()
def change_field_display_name(resource, field_keyname, new_displayname, make_transaction=False):
if not isinstance(resource, VectorLayer):
raise Exception('Unsupported resource type!')
if field_keyname not in [field.keyname for field in resource.fields]:
raise Exception('Field does not exists in the table!')
#start transaction
if make_transaction:
transaction.manager.begin()
for field in resource.fields:
if field.keyname == field_keyname:
field.display_name = new_displayname
#close transaction
if make_transaction:
transaction.manager.commit()
else:
db_session = DBSession()
db_session.flush()
def change_field_datatype(resource, field_keyname, new_field_type, make_transaction=False):
if not isinstance(resource, VectorLayer):
raise Exception('Unsupported resource type!')
if new_field_type not in FIELD_TYPE.enum:
raise Exception('Unsupported field type!')
target_field = filter(lambda field: field.keyname == field_keyname, resource.fields)
if not target_field:
raise Exception('Field not found in the table!')
else:
target_field = target_field[0]
if new_field_type == target_field.datatype:
print 'Field already has such type!'
return
#start transaction
if make_transaction:
transaction.manager.begin()
# change data type physic
VectorLayerUpdater.__change_column_datatype(resource.tbl_uuid, target_field.fld_uuid, _FIELD_TYPE_2_DB[new_field_type])
# set new type in field registry
target_field.datatype = new_field_type
target_field.persist()
#close transaction
if make_transaction:
transaction.manager.commit()
else:
db_session = DBSession()
db_session.flush()
def append_field(resource, field_keyname, field_type, field_display_name, field_grid_vis=True, make_transaction=False):
if not isinstance(resource, VectorLayer):
raise Exception('Unsupported resource type!')
if field_type not in FIELD_TYPE.enum:
raise Exception('Unsupported field type!')
if field_keyname in [field.keyname for field in resource.fields]:
raise Exception('Field already exists in the table!')
#start transaction
if make_transaction:
transaction.manager.begin()
#create uuid for field
uid = str(uuid.uuid4().hex)
#create column
VectorLayerUpdater.__create_column(resource.tbl_uuid, uid, _FIELD_TYPE_2_DB[field_type])
#add field to register
vfl = VectorLayerField()
vfl.keyname = field_keyname
vfl.datatype = field_type
vfl.display_name = field_display_name
vfl.grid_visibility = field_grid_vis
vfl.fld_uuid = uid
resource.fields.append(vfl)
#close transaction
if make_transaction:
transaction.manager.commit()
else:
db_session = DBSession()
db_session.flush()
def init_calendar():
print('Fill calendar...')
db_session = DBSession()
db_session.autoflush = False
count = db_session.query(Calendar).count()
if count != 0:
print ' Calendar is not empty! Returning...'
return
start_date = date(2014, 1, 1)
max_date = date(2025, 1, 1)
active_date = start_date
quarter_names = {
1: u'1 кв.',
2: u'2 кв.',
3: u'3 кв.',
4: u'4 кв.',
}
month_names = {
1: u'Январь',
2: u'Февраль',
3: u'Март',
4: u'Апрель',
5: u'Май',
6: u'Июнь',
7: u'Июль',
8: u'Август',
9: u'Сентябрь',
10: u'Октябрь',
11: u'Ноябрь',
12: u'Декабрь',
}
week_day_names = {
1: u'Понедельник',
2: u'Вторник',
3: u'Среда',
4: u'Четверг',
5: u'Пятница',
6: u'Суббота',
7: u'Воскресенье',
}
week_day_short_names = {
1: u'Пн',
2: u'Вт',
3: u'Ср',
4: u'Чт',
5: u'Пт',
6: u'Сб',
7: u'Вс',
}
relat_day = relativedelta.relativedelta(days=+1)
while active_date < max_date:
cal = Calendar()
cal.id = active_date.year * 10000 + active_date.month * 100 + active_date.day
cal.full_date = active_date
cal.year_number = active_date.year
cal.semester_number = 1 if active_date.month < 7 else 2
cal.semester_name = u'1 полугодие' if active_date.month<7 else u'2 полугодие'
cal.quarter_number = (active_date.month-1)//3 + 1
cal.quarter_name = quarter_names[cal.quarter_number]
cal.month_number = active_date.month
cal.month_name = month_names[active_date.month]
cal.year_week_number = active_date.isocalendar()[1]
cal.month_week_number = get_week_of_month(active_date)
cal.month_decade_number = (active_date.day < 11) * 1 + \
(11 <= active_date.day <= 20) * 2 + \
(active_date.day > 20) * 3
cal.year_day_number = active_date.timetuple().tm_yday
cal.month_day_number = active_date.day
cal.week_day_number = active_date.weekday() + 1
cal.week_day_name = week_day_names[cal.week_day_number]
cal.week_day_short_name = week_day_short_names[cal.week_day_number]
cal.weekend = cal.week_day_number > 5
cal.persist()
active_date = active_date + relat_day
db_session.flush()
def run(cls):
db_session = NgwSession()
transaction.manager.begin()
# clear all data from tables
for table in [BuiltFosc, BuiltCable, BuiltOpticalCross, BuiltAccessPoint, BuiltSpecTransition]:
db_session.query(table).delete()
db_session.flush()
# get dicts
fosc_types = {x.type: x for x in db_session.query(FoscType).all()}
cable_laying_methods = {x.method: x for x in db_session.query(CableLayingMethod).all()}
optical_cross_types = {x.type: x for x in db_session.query(OpticalCrossType).all()}
ap_types = {x.type: x for x in db_session.query(AccessPointType).all()}
spec_laying_methods = {x.method: x for x in db_session.query(SpecLayingMethod).all()}
# get all focls
fs_resources = db_session.query(FoclStruct).all()
for fs in fs_resources:
# fosc handler
fact_lyr = cls.get_layer_by_type(fs, 'actual_real_fosc')
if fact_lyr:
# get all rows and aggregate
fosc_values = []
query = fact_lyr.feature_query()
for feat in query():
if feat.fields['built_date']:
fosc_values.append((feat.fields['built_date'].date(), feat.fields['type_fosc']))
fosc_aggregation = {k: len(list(g)) for k, g in itertools.groupby(sorted(fosc_values))}
# save to table
for (build_dt, fs_type), count in fosc_aggregation.iteritems():
row = BuiltFosc()
row.resource_id = fs.id
row.fosc_count = count
if fs_type:
if fs_type not in fosc_types.keys():
ot = FoscType()
ot.type = fs_type
ot.persist()
fosc_types[fs_type] = ot
row.fosc_type = fosc_types[fs_type]
row.build_date = db_session.query(Calendar).filter(Calendar.full_date==build_dt).one()
row.persist()
db_session.flush()
# cross handler
fact_lyr = cls.get_layer_by_type(fs, 'actual_real_optical_cross')
if fact_lyr:
# get all rows and aggregate
cross_values = []
query = fact_lyr.feature_query()
for feat in query():
if feat.fields['built_date']:
cross_values.append((feat.fields['built_date'].date(), feat.fields['type_optical_cross']))
cross_aggregation = {k: len(list(g)) for k, g in itertools.groupby(sorted(cross_values))}
# save to table
for (build_dt, cross_type), count in cross_aggregation.iteritems():
row = BuiltOpticalCross()
row.resource_id = fs.id
row.optical_cross_count = count
if cross_type:
if cross_type not in optical_cross_types.keys():
ot = OpticalCrossType()
ot.type = cross_type
ot.persist()
optical_cross_types[cross_type] = ot
row.optical_cross_type = optical_cross_types[cross_type]
row.build_date = db_session.query(Calendar).filter(Calendar.full_date==build_dt).one()
row.persist()
db_session.flush()
# ap handler
fact_lyr = cls.get_layer_by_type(fs, 'actual_real_access_point')
if fact_lyr:
# get all rows and aggregate
ap_values = []
query = fact_lyr.feature_query()
for feat in query():
if feat.fields['built_date']:
ap_values.append((feat.fields['built_date'].date(), None))
ap_aggregation = {k: len(list(g)) for k, g in itertools.groupby(sorted(ap_values))}
# save to table
for (build_dt, ap_type), count in ap_aggregation.iteritems():
row = BuiltAccessPoint()
row.resource_id = fs.id
row.access_point_count = count
if ap_type:
if ap_type not in ap_types.keys():
apt = AccessPointType()
apt.type = ap_type
apt.persist()
ap_types[ap_type] = apt
row.access_point_type = ap_types[ap_type]
row.build_date = db_session.query(Calendar).filter(Calendar.full_date==build_dt).one()
row.persist()
db_session.flush()
# cabling handler
fact_lyr = cls.get_layer_by_type(fs, 'actual_real_optical_cable')
if fact_lyr:
# get all rows and aggregate
cable_values = []
query = fact_lyr.feature_query()
query.geom_length()
for feat in query():
if feat.fields['built_date']:
cable_values.append((feat.fields['built_date'].date(), feat.fields['laying_method'], feat.calculations['geom_len']))
cable_aggregation = {k: sum(x[2] for x in list(g)) for k, g in itertools.groupby(sorted(cable_values), key=lambda x: (x[0], x[1]))}
# save to table
for (build_dt, lay_meth), cable_len in cable_aggregation.iteritems():
row = BuiltCable()
row.resource_id = fs.id
row.cable_length = round(cable_len/1000.0, 3) if cable_len else 0
if lay_meth:
if lay_meth not in cable_laying_methods.keys():
lm = CableLayingMethod()
lm.method = lay_meth
lm.persist()
cable_laying_methods[lay_meth] = lm
row.laying_method = cable_laying_methods[lay_meth]
row.build_date = db_session.query(Calendar).filter(Calendar.full_date==build_dt).one()
row.persist()
db_session.flush()
# spec trans handler
fact_lyr = cls.get_layer_by_type(fs, 'actual_real_special_transition')
if fact_lyr:
# get all rows and aggregate
spec_trans_values = []
query = fact_lyr.feature_query()
query.geom_length()
for feat in query():
if feat.fields['built_date']:
spec_trans_values.append((feat.fields['built_date'].date(), feat.fields['special_laying_method'], feat.calculations['geom_len']))
spec_trans_aggregation = {k: list(g) for k, g in itertools.groupby(sorted(spec_trans_values), key=lambda x: (x[0], x[1]))}
# save to table
for (build_dt, spec_lay_meth), specs in spec_trans_aggregation.iteritems():
row = BuiltSpecTransition()
row.resource_id = fs.id
length = sum(x[2] for x in specs)
row.spec_trans_length = round(length/1000.0, 3) if length else 0
row.spec_trans_count = len(specs)
if spec_lay_meth:
if spec_lay_meth not in spec_laying_methods.keys():
slm = SpecLayingMethod()
slm.method = spec_lay_meth
slm.persist()
spec_laying_methods[spec_lay_meth] = slm
row.spec_laying_method = spec_laying_methods[spec_lay_meth]
row.build_date = db_session.query(Calendar).filter(Calendar.full_date==build_dt).one()
row.persist()
db_session.flush()
transaction.manager.commit()
