def fdump(self, opts):
print("te.fdump")
data = list()
kw = dict()
qs = Transaction.objects.all()
if 'tstamp' in opts:
dt = myutils.unixtime2dt(opts['tstamp'])
qs = qs.filter(created__gre=dt)
for f in [
'machine', 'model_name', 'rid', 'prid', 'srid', 'name', 'value'
]:
if not f in opts:
continue
if isinstance(opts[f], list):
kw[f + '__in'] = opts[f]
else:
kw[f] = opts[f]
qs = qs.filter(**kw)
for t in qs:
data.append(t.data())
return data
def reinit(cls):
print("deleted", cls.objects.all().delete())
dt = myutils.unixtime2dt(0)
for srv in cls.srv:
ts = TransactionServer(machine=srv['machine'],
url=srv['url'],
private=srv['private'],
netprocess=srv['netprocess'],
lastupdate=dt)
print(ts)
ts.save()
def restore_helper(self, o, data):
skipfields = ['id']
rid = data['rid']
print(data)
if isinstance(o, ModelBase):
print("find by rid:", rid)
try:
p = o.objects.get(rid=rid)
except ObjectDoesNotExist:
print("Create", o)
o = o()
# now o is always instance
print("restore rid", rid)
for f in o._meta.get_fields():
if f.name in skipfields:
continue
if not f.is_relation:
if f.name in data:
if isinstance(f, models.fields.DateTimeField) and data[
f.name] is not None:
setattr(o, f.name, myutils.unixtime2dt(data[f.name]))
else:
# no datetime, simple field
setattr(o, f.name, data[f.name])
else:
print("missing key {} in data".format(f.name))
# print "{} = {}".format(f.name,repr(getattr(o,f.name)))
elif isinstance(f, models.ForeignKey):
relmodel = f.related_model
relname = f.related_model.__name__
# try to get relrid
# maybe in parents
rel_instance = self.rid2instance(f.related_model, None,
data['_parents'])
print("FK to {} {} {}".format(repr(relname), relmodel,
rel_instance))
print(data)
setattr(o, f.name, rel_instance)
return o
def load_transaction(self, o, t, fmethod=None):
def trans(o, fmethod, name, value):
mname = o.__class__.__name__
# print "TRANS {} {} : {} ({}={})".format(mname, o, fmethod, name, value)
kwargs = dict()
kwargs['__'.join(fmethod[1:])] = value
try:
rel_o = o._meta.get_field(name).related_model.objects.get(
**kwargs)
except ObjectDoesNotExist as e:
raise TransactionWarning("{} {} {}={} EXCEPTION: {}".format(
mname, fmethod, name, value, str(e)))
return rel_o
if isinstance(fmethod, tuple):
if fmethod[0] == 'DT':
# print "DT {} = {}".format(t['name'],repr(t['value']))
if t['value'] is None:
setattr(o, t['name'], None)
else:
setattr(o, t['name'], myutils.unixtime2dt(int(t['value'])))
elif fmethod[0] == 'INT':
setattr(o, t['name'], int(t['value']))
elif fmethod[0] == 'BOOL':
try:
val = json.loads(t['value'])
except ValueError as e:
print("json decode error. value: {}".format(
repr(t['value'])))
setattr(o, t['name'], val)
elif fmethod[0] == 'ignore':
pass
elif fmethod[0] == 'trans':
value = trans(o, fmethod, t['name'], t['value'])
setattr(o, t['name'], value)
else:
raise TransactionError(
'load_transaction: unknown fmethod {}'.format(
repr(fmethod[0])))
elif fmethod is None:
setattr(o, t['name'], t['value'])
else:
raise TransactionError('unknown type of field method {}'.format(
repr(fmethod)))
def dump(self, timestamp=None, srid=None):
data = list()
if srid is not None:
qs = Transaction.objects.filter(srid=srid)
elif timestamp is not None:
dt = myutils.unixtime2dt(timestamp)
qs = Transaction.objects.filter(created__gte=dt)
else:
qs = Transaction.objects.all()
for t in qs:
data.append(t.data())
# hier = ['srid','model_name','rid']
# return list2dict(data, hier)
return data
def sync(self, opts=None, ro=False, verbosity=1, skip=False):
te = TransactionEngine()
if opts is None:
ls = myutils.dt2unixtime(self.lastupdate)
url = self.action_url('sync', ls)
else:
url = self.action_url('fsync', opts)
if verbosity >= 2:
print("URL:", url)
r = requests.get(url)
if r.status_code != 200:
print("ERROR! Couldn't get from {} (HTTP code: {})".format(
url, r.status_code))
return
data = json.loads(r.text)
last = 0
for t in data:
if t['created'] > last:
last = t['created']
print("got", len(data), "transactions from", self.machine)
#with django.db.transaction.atomic():
msg = te.load(data, ro=ro, verbosity=verbosity, skip=skip)
if opts is None and not ro:
# update lastupdate only in real run
self.lastupdate = myutils.unixtime2dt(last)
return msg
class TransactionServer(models.Model):
lastupdate = models.DateTimeField(default=myutils.unixtime2dt(0))
machine = models.CharField(max_length=100, default='')
url = models.CharField(max_length=100, default='')
netprocess = models.BooleanField(default=True)
private = models.BooleanField(default=False)
srv = [
{
'machine': 'alpha',
'url': 'https://alpha.okerr.com/',
'netprocess': True,
'private': False
},
{
'machine': 'braconnier',
'url': 'http://localhost:8000/',
'netprocess': False,
'private': True
},
{
'machine': 'charlie',
'url': 'https://charlie.okerr.com/',
'netprocess': True,
'private': False
},
]
def __unicode__(self):
return "{} {} {}".format(self.machine, self.url, self.lastupdate)
# transactionserver.action_url
def action_url(self, action, *args):
if action == 'sync':
urltpl = urllib.parse.urljoin(self.url, '/api/sync/{}')
elif action == 'sdump':
urltpl = urllib.parse.urljoin(self.url, '/api/sdump/{}')
elif action == 'setci':
urltpl = urllib.parse.urljoin(self.url, '/api/setci')
elif action == 'fsync':
url = '/api/fsync'
for a, v in args[0].iteritems():
if isinstance(v, list):
for vv in v:
url += '/{}={}'.format(a, vv)
else:
url += '/{}={}'.format(a, v)
url = urllib.parse.urljoin(self.url, url)
return url
else:
print("unknown action", repr(action))
return None
# print "tpl: {} args: {}".format(urltpl, list(args))
url = urltpl.format(*args)
return url
# transactionserver.ciserver
@staticmethod
def ciserver(ci=None):
if ci is None:
ci = myci()
sname = settings.CLUSTER[ci]
tsrv = TransactionServer.objects.get(machine=sname)
return tsrv
# transactionserver.reinit
@classmethod
def reinit(cls):
print("deleted", cls.objects.all().delete())
dt = myutils.unixtime2dt(0)
for srv in cls.srv:
ts = TransactionServer(machine=srv['machine'],
url=srv['url'],
private=srv['private'],
netprocess=srv['netprocess'],
lastupdate=dt)
print(ts)
ts.save()
# transactionserver.sync_map
@classmethod
def sync_map(cls, ro=False, verbosity=1):
# print "sync map",settings.HOSTNAME
out = dict()
if ro:
print("read-only")
if settings.HOSTNAME in settings.SYNC_MAP:
for machine in settings.SYNC_MAP[settings.HOSTNAME]:
print("sync", settings.HOSTNAME, "from", machine)
tsrv = cls.objects.get(machine=machine)
msg = tsrv.sync(ro=ro, verbosity=verbosity)
out[tsrv.machine] = msg
tsrv.save()
return out
# transactionserver.cron
@classmethod
def cron(cls):
# return cls.sync_map()
return None
# transactionserver.sync
def sync(self, opts=None, ro=False, verbosity=1, skip=False):
te = TransactionEngine()
if opts is None:
ls = myutils.dt2unixtime(self.lastupdate)
url = self.action_url('sync', ls)
else:
url = self.action_url('fsync', opts)
if verbosity >= 2:
print("URL:", url)
r = requests.get(url)
if r.status_code != 200:
print("ERROR! Couldn't get from {} (HTTP code: {})".format(
url, r.status_code))
return
data = json.loads(r.text)
last = 0
for t in data:
if t['created'] > last:
last = t['created']
print("got", len(data), "transactions from", self.machine)
#with django.db.transaction.atomic():
msg = te.load(data, ro=ro, verbosity=verbosity, skip=skip)
if opts is None and not ro:
# update lastupdate only in real run
self.lastupdate = myutils.unixtime2dt(last)
return msg
# transactionserver.ssync
def ssync(self, srid, ro=False, verbose=False):
"""
sync only one SRID (profile)
"""
te = TransactionEngine()
url = self.action_url('sdump', srid)
r = requests.get(url)
if r.status_code != 200:
print("ERROR! Couldn't get from {} (HTTP code: {})".format(
url, r.status_code))
return
data = json.loads(r.text)
print("got", len(data), "transactions from", self.machine)
with django.db.transaction.atomic():
te.load(data, ro=ro)
# transactionserver.setci
def setci(self, ci, email):
data = {'ci': ci, 'email': email}
url = self.action_url('setci', email)
r = requests.post(url, data)
if r.status_code != 200:
print("Error! status: {} url: {}".format(r.status_code, url))
else:
print("remote:", r.text)
# transactionserver.take
def take(self, srid):
ci = myci()
self.setci(ci, srid)
self.ssync(srid)
def d2o(self, d, model, parents=None):
debug = False
debugmodels = []
if model.__name__ in debugmodels:
debug = True
if debug:
print("d2o {}".format(model.__name__))
def backtrans(mname, fname, trans,value,parents):
#print "backtrans mname: {} fname: {} trans: {} value: {} parents: {}".format(
# mname, fname, trans, value, parents
# )
if value is None:
return None
field = trans.split(':')[1].split('.')[1]
f = model._meta.get_field(fname)
rmodel = f.related_model
kw = dict()
kw[field] = value
if trans.startswith('ptrans'):
pmname = trans.split(':')[2]
#print "pmname",pmname
rfname = self.get_fk_field_to(rmodel, parents[pmname].__class__)
#print "rfname:",rfname
kw[rfname] = parents[pmname]
#print "rmodel:", rmodel
#print "backtrans rmodelget:", kw
try:
ri = rmodel.objects.get(**kw)
except ObjectDoesNotExist as e:
log.error('EXCEPTION {}: {}. kw: {}'.format(type(e), e, str(kw)))
return ri
mname = model.__name__
if parents is None:
parents = dict()
# self.learn(model)
o = model()
m = self.models[mname]
for fname, ftype in m['fields'].items():
if ftype is None:
#print "Set {} = {}".format(fname, d[fname])
# with this check we can add new fields
if fname in d:
setattr(o, fname, d[fname])
elif ftype == 'ignore':
pass
elif ftype == 'DT':
if fname in d:
val = d[fname]
else:
val = None
if val is None:
setattr(o, fname, None)
else:
setattr(o, fname, myutils.unixtime2dt(d[fname]))
elif ftype == 'parent':
#print "procces {}: {}.{} parents: {}".format(mname, fname, ftype, parents)
#print json.dumps(m, indent=4)
f = model._meta.get_field(fname)
setattr(o,fname, self.get_parent(model, fname, parents))
elif ftype.startswith('trans:') or ftype.startswith('ptrans:'):
# print "{} {} {}".format(ftype, d, fname)
setattr(o, fname, backtrans(mname, fname, ftype, d[fname], parents))
else:
raise ValueError("d2o: dont know how to handle ftype {} for {}.{}".format(ftype, mname, fname))
if m['trans_last_sync']:
o.trans_last_sync = timezone.now()
if m['post_import']:
# print "PI",mname
self.vprint(3, "post_import {}: {}".format(mname,o))
o.post_import(d)
else:
if m['__bulk']:
self.bulk_save_add(o)
else:
# print "SAVE",mname,o
o.save()
if not mname in self.processed:
self.processed[mname] = 1
else:
self.processed[mname] += 1
#if m['reanimate']:
# self.reanimate.append(o)
chparents = dict(parents)
if mname in chparents:
raise ValueError
chparents[mname] = o
# addparent
for pfield in m['__addparent']:
p = getattr(o, pfield)
pmname = p.__class__.__name__
if pmname in chparents:
raise ValueError
chparents[pmname] = p
for chs in m['__children']:
# print "Process children structure:",json.dumps(chs, indent=4)
chmname = chs['model']
chm = self.get_model(chmname)
if chmname in d:
for chres in d[chmname]:
# print chres
# self.d2o(chres, chm, chparents)
self.delay_d2o(chmname, chres, chparents)
if debug:
print("d2o finished:", o)