def clean_model(self, model, days, count, verbose=False):
from uliweb import functions
import time
import logging
import types
from uliweb.orm import Begin, Commit, Rollback
log = logging.getLogger(__name__)
if verbose:
print('Clean {}, days={}, count={} ...'.format(model, days, count))
M = functions.get_model(model)
if hasattr(M, 'clear_data'):
b = time.time()
t = 0
Begin()
try:
ret = M.clear_data(days, count)
Commit()
except Exception as e:
Rollback()
log.exception(e)
return
if isinstance(ret, types.GeneratorType):
while 1:
Begin()
try:
n = ret.next()
if n == None:
break
t += n
Commit()
except StopIteration:
break
except Exception as e:
Rollback()
log.exception(e)
break
else:
t = ret
print(
'Used {} seconds to clean the {}, total records is {}'.format(
time.time() - b, model, t))
else:
print('There is no clear_data() function defined for {}'.format(
model))
def test(i):
from uliweb.orm import get_model
from uliweb.orm import Begin, Commit, Rollback
try:
Begin()
TaskDB = get_model('workflow_task')
cond = None
query = TaskDB.filter(cond).limit(10)
ids = [task_obj.id for task_obj in query]
Commit(close=True)
except:
Rollback()
raise
from uliweb.orm import print_pool_status
print_pool_status()
from random import randint
if randint(1,2) == 1:
test2()
else:
import gevent
gevent.sleep(0.5)
print i
return
def test_manual_rollback():
from uliweb.orm import Begin, Commit, Rollback
Begin()
Blog = functions.get_model('blog')
b = Blog(title='test_add', content='test_add')
b.save()
Rollback()
return 'ok'
def start_job(job, now=None):
from uliweb.utils import date
now = now or date.now()
log.info(u'Starting job [{title} -- {time}] at {now}'.format(
title=job.title, now=now, time=job.time))
Task = functions.get_model('cron_task')
Begin()
try:
parent = functions.AsyncCommand(
'',
title=job.title,
startup_time=now,
category='cron',
message_source='l',
timeout=job.timeout,
#todo add correlation to job
)
commands = []
ids = {}
#process task
for row in Task.filter(Task.c.cron_job == job.id):
#process async task
d = parse_command(row)
c = functions.AsyncCommand(
title=row.label,
category='cron',
message_source='l',
correlation=job.id,
queue=(row.queue or 'default').split(','),
timeout=row.timeout or None,
# correlation_link='/async_task/view/{0}?next=/cron/{1}'.format(job.id, obj.id)
**d)
commands.append((c, row.depend_tasks))
ids[row.id] = c.task_id
parent.add_child(c)
#fix depends
for c, depends in commands:
_d = [ids[x] for x in depends]
c.depend_tasks = _d
functions.call_async(parent)
job.instances.add(parent.task_id)
Commit()
except Exception as e:
Rollback()
log.exception(e)
raise
def init(self):
super(InsertWorker, self).init()
from uliweb import settings
from uliweb.orm import Reset, get_session
patch_mysqldb()
get_session(create=True)
# Reset()
# table = reflect_table(self._table)
para = dict(zip(self.fields, [''] * len(self.fields)))
self.sql = str(self.table.insert().values(para))
# self.sql = self.model.table.insert()
Begin()
def save(self, **kwargs):
from uliweb.orm import Begin, Commit, Rollback
Begin()
try:
self.task.update(**kwargs)
self._new_data = {}
ret = self.task.save(version=True, saved=self._saved)
Commit()
if self._new_data:
self.on_changed(self._new_data)
except Exception as e:
Rollback()
self.log.exception(e)
raise
def scan_step_decide_all_dir(id=1):
from uliweb import settings
ScanPathes = get_model("scanpathes")
root_dp = settings.SCAN.DIR
def get_path(id):
r = do_(select(ScanPathes.c, ScanPathes.c.id == id))
return r.fetchone()
def get_children_pathes(id):
r = do_(select(ScanPathes.c, ScanPathes.c.parent == id))
return r.fetchall()
def decide_path(id):
path = get_path(id)
if path.type == 'f':
return path.crbits or 0, path.crindex_bits or 0
else:
#print "dir %d crtype ?"%(id)
crbits = 0x00
crindex_bits = 0x00
for p in get_children_pathes(id):
crbits_child, crindexbits_child = decide_path(p.id)
#print "\t",p.path
#print "\t%d|%d="%(crindex_bits,crindexbits_child),
crbits |= crbits_child
crindex_bits |= crindexbits_child
#print crindex_bits
crtype = crbits2crtype(crbits)
if (path.crtype != crtype) or (path.crbits != crbits) or (
path.crindex_bits != crindex_bits):
do_(ScanPathes.table.update().where(
ScanPathes.c.id == path.id).values(
crtype=crtype,
crbits=crbits,
crindex_bits=crindex_bits))
print "%s\t0x%04x\t0x%08x\t%s" % (crtype2csstag(crtype)[3:8],
crbits, crindex_bits, path.path)
return crbits, crindex_bits
print "want to decide %d" % (id)
Begin()
decide_path(id)
Commit()
path = get_path(id)
print path.path, path.crindex_bits, path.crbits
def load_ready_tasks(self, queue):
from uliweb.orm import Begin, Commit, Rollback, Reset
from gevent.queue import Full
import gevent
if self.queue.full():
gevent.sleep(0.3)
return
Begin()
try:
cond = self.task_model.c.state == 2 # READY
cond = (self.task_model.c.async_status == 0
or self.task_model.c.async_status == None) & cond
query = self.task_model.filter(cond)
query = query.order_by(self.task_model.c.async_deliver_date.asc())
query = query.limit(self.limit)
isFull = False
isEmpty = True
for item in query:
self.tid = self.tid + 1
try:
queue.put_nowait([self.tid, item.id])
item.update(async_status=1)
item.save()
isEmpty = False
except Full:
isFull = True
break
Commit()
if isEmpty and self.debug:
self.prints(">>> [DEBUG] No tasks found %s" %
self.gettimestamp())
if isFull:
gevent.sleep(0.3)
if isEmpty:
gevent.sleep(0.3)
except:
Rollback()
import traceback
traceback.print_exc()
def process_request(self, request):
Begin()
def process_request(self, request):
set_echo(False)
Begin()
def scan_step_all_path():
from uliweb import settings
ScanPathes = get_model("scanpathes")
ScanPathes.remove()
allext = {}
root_dp = settings.SCAN.DIR
ScanPathes(
path=".",
type="d",
).save()
count = 0
Begin()
IGNORE_DIRS_SET = set(settings.SCAN.DIR_IGNORE)
for root, dirs, files in os.walk(root_dp):
root_relp = os.path.relpath(root, root_dp)
if not isinstance(root_relp, unicode):
root_relp = root_relp.decode("utf8")
sys.stdout.write(".")
rp = ScanPathes.get(ScanPathes.c.path == root_relp)
if not rp:
print "\ncan not find in db so do not scan %s" % (root)
continue
ignore_dirs = []
for dn in dirs:
dp = os.path.join(root, dn)
if os.path.islink(dp):
print "\nignore link:%s" % (dp)
ignore_dirs.append(dn)
elif dn in IGNORE_DIRS_SET:
print "\nignore dir: %s" % (dp)
ignore_dirs.append(dn)
else:
relp = os.path.relpath(dp, root_dp)
do_(ScanPathes.table.insert().values(path=relp.decode("utf8"),
type="d",
parent=rp.id))
for dn in ignore_dirs:
dirs.remove(dn)
l = root.split(os.sep)
for fn in files:
fp = os.path.join(root, fn)
if not os.path.islink(fp):
p, ext = os.path.splitext(fn)
relp = os.path.relpath(fp, root_dp)
do_(ScanPathes.table.insert().values(path=relp.decode("utf8"),
type="f",
ext=ext,
parent=rp.id))
if allext.has_key(ext):
allext[ext] += 1
else:
allext[ext] = 1
else:
print "\nignore link:%s" % (fp)
Commit()
Exts = get_model("exts")
Exts.remove()
for i, k in enumerate(allext):
Exts(ext=k, num=allext[k]).save()
print
def scan_step_all_copyright():
import re
from uliweb import settings
exts_ignore_dict = {}
for ext in settings.SCAN.FILE_EXTS_IGNORE:
exts_ignore_dict[ext] = True
ScanPathes = get_model("scanpathes")
CopyrightInfo = get_model('copyrightinfo')
CopyrightInfo.remove()
files = ScanPathes.filter(ScanPathes.c.type == 'f')
restring = get_restring_from_relist(settings.SCAN.RE_LIST)
cobj_copyright = re.compile(restring, re.M)
root_dp = settings.SCAN.DIR
count = 0
nback = 0
tnum = ScanPathes.filter(ScanPathes.c.type == 'f').count()
Begin()
for path in files:
if not exts_ignore_dict.has_key(path.ext):
fp = os.path.join(root_dp, path.path)
f = open(fp)
c = f.read()
f.close()
crbits = 0
crbits_not = 0
isbin = False
crindex_bits = 0
cribegin = -1
criend = -1
if c[:4] == '\x7fELF':
isbin = True
elif c[:8] == '!<arch>\n':
isbin = True
elif c[:6] == 'CATI\x01\x00':
isbin = True
else:
for ch in c:
if ch == '\0':
isbin = True
break
#if isbin:
# print "%s is binary"%(path.path)
if not isbin:
for m in cobj_copyright.finditer(c):
d = m.groupdict()
for i, k in enumerate(d):
if d[k] != None:
index = int(k[1:])
crindex_bits |= (0x01 << index)
new_crbits, not_flag = index2crbits(
k, settings.SCAN.RE_LIST)
if not_flag:
crbits_not |= new_crbits
else:
crbits |= new_crbits
ibegin = m.start(0)
iend = m.end(0)
do_(CopyrightInfo.table.insert().values(
path=path.id,
crindex=index,
ibegin=ibegin,
iend=iend,
))
if cribegin < 0 or ibegin < cribegin:
cribegin = ibegin
if criend < 0 or iend > criend:
criend = iend
if crbits_not:
crbits = crbits & (crbits_not ^ CRBITS_ALL)
crtype = crbits2crtype(crbits)
do_(ScanPathes.table.update().where(
ScanPathes.c.id == path.id).values(
copyright=((crbits & CRBITS_COPYRIGHT) != 0),
copyright_inhouse=((crbits & CRBITS_COPYRIGHT_INHOUSE) !=
0),
copyright_gpl=((crbits & CRBITS_COPYRIGHT_GPL) != 0),
copyright_oos=((crbits & CRBITS_COPYRIGHT_OOS) != 0),
crbits=crbits,
crtype=crtype,
crindex_bits=crindex_bits,
cribegin=cribegin,
criend=criend))
count += 1
s = "%d/%d" % (count, tnum)
out = "%s%s" % (nback * "\b", s)
print out,
sys.stdout.flush()
nback = len(s) + 1
Commit()