def insert_data(self, query, args):
self.logger.debug("Inserting logs to searchd")
result = None
for _ in 1, 2, 3:
try:
c = self.conn.cursor()
result = c.execute(query, args)
self.logger.debug("%s rows inserted" % c.rowcount)
c.close()
except ProgrammingError:
self.logger.exception(
"Can't insert values to index: %s" % query)
except DatabaseError as e:
self.logger.exception("Sphinx connection error: %s" % e)
try:
close_old_connections()
except Exception as e:
self.logger.exception("Can't reconnect: %s" % e)
os.kill(os.getpid(), signal.SIGKILL)
except Exception:
self.logger.exception("Unhandled error in insert_data")
else:
return result
self.logger.error("Can't insert data in 3 tries, exit process")
os.kill(os.getpid(), signal.SIGKILL)
def handle(self, *args, **options):
"""
Iterates over all the CRON_CLASSES (or if passed in as a commandline argument)
and runs them.
"""
cron_classes = options['cron_classes']
if cron_classes:
cron_class_names = cron_classes
else:
cron_class_names = getattr(settings, 'CRON_CLASSES', [])
try:
crons_to_run = [get_class(x) for x in cron_class_names]
except Exception:
error = traceback.format_exc()
self.stdout.write('Make sure these are valid cron class names: %s\n%s' % (cron_class_names, error))
return
for cron_class in crons_to_run:
run_cron_with_cache_check(
cron_class,
force=options['force'],
silent=options['silent']
)
clear_old_log_entries()
close_old_connections()
def _get_instances(cluster):
try:
instances.extend(cluster.get_instances())
except (GanetiApiError, Exception):
pass
finally:
close_old_connections()
def get_user(self, username):
try:
u = User.objects.get(username=username)
return u
except:
db.close_old_connections()
raise
def handle(self, *args, **options):
try:
os.chdir(settings.BASE_DIR)
graceful_startup()
session = SmSession.objects.get_or_create()[0]
if session.update_on_startup:
session.update_available = False
session.save()
close_old_connections()
command = git + ' stash'
subprocess.call(command, shell=True) # trying to get rid of settings.sqlite3 change
command = git + ' reset --hard'
subprocess.call(command, shell=True)
command = git + ' pull'
git_status = subprocess.check_output(command, shell=True)
# TODO: Make sure the pull actually worked
except:
print "Failed to update!"
try:
command = git + ' reset'
subprocess.call(command, shell=True)
command = git + ' stash apply'
subprocess.call(command, shell=True)
except:
print "Failed to gracefully reset!"
return False
finally:
session = SmSession.objects.get_or_create()[0]
session.update_on_startup = False
session.save()
def handle(self, *args, **options):
close_old_connections()
db = options.get('database')
connection = connections[db]
interactive = options.get('interactive')
settings = dict(connection.settings_dict)
name = settings['NAME']
if interactive and self.should_ask:
msg = ''.join([self.message % {
'name': name,
}, self.ask_message])
confirm = input(msg)
else:
confirm = self.default
if confirm == 'yes':
backend = get_backend(settings)
backend.connect()
try:
self.execute_sql(backend, **options)
except Exception as e:
six.reraise(CommandError, CommandError(self.error_message % {
'name': name,
'error': e,
}), sys.exc_info()[2])
finally:
backend.close()
self.post_execute(**options)
def schedule_changed(self):
try:
close_old_connections()
# If MySQL is running with transaction isolation level
# REPEATABLE-READ (default), then we won't see changes done by
# other transactions until the current transaction is
# committed (Issue #41).
try:
transaction.commit()
except transaction.TransactionManagementError:
pass # not in transaction management.
last, ts = self._last_timestamp, self.Changes.last_change()
except DatabaseError as exc:
logger.exception('Database gave error: %r', exc)
return False
except InterfaceError:
warning(
'DatabaseScheduler: InterfaceError in schedule_changed(), '
'waiting to retry in next call...'
)
return False
try:
if ts and ts > (last if last else ts):
return True
finally:
self._last_timestamp = ts
return False
def save_task(task, broker):
"""
Saves the task package to Django or the cache
"""
# SAVE LIMIT < 0 : Don't save success
if not task.get('save', Conf.SAVE_LIMIT > 0) and task['success']:
return
# async next in a chain
if task.get('chain', None):
tasks.async_chain(task['chain'], group=task['group'], cached=task['cached'], sync=task['sync'], broker=broker)
# SAVE LIMIT > 0: Prune database, SAVE_LIMIT 0: No pruning
db.close_old_connections()
try:
if task['success'] and 0 < Conf.SAVE_LIMIT <= Success.objects.count():
Success.objects.last().delete()
Task.objects.create(id=task['id'],
name=task['name'],
func=task['func'],
hook=task.get('hook'),
args=task['args'],
kwargs=task['kwargs'],
started=task['started'],
stopped=task['stopped'],
result=task['result'],
group=task.get('group'),
success=task['success'])
except Exception as e:
logger.error(e)
def check_simulation_version():
"""Will return None if the database is not usable"""
close_old_connections()
version = None
try:
executable = adsm_executable_command()[0]
process = subprocess.Popen(executable + " --version --silent", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
output, error = process.communicate(timeout=200)
exit_code = process.returncode
except:
exit_code = 1
output = None
try:
process.kill()
except:
pass
if exit_code == 0:
version = output.splitlines()[-1].decode()
except:
print("Unable to get Simulation Version!")
return None
try:
SmSession.objects.all().update(simulation_version=version)
except:
print("Unable to store Simulation Version!")
return version
def reset_db(name, fail_ok=True):
"""Resets the database to a healthy state by any means necessary. Tries to delete the file, or flush all tables, then runs the migrations from scratch.
It now checks if the file exists. This will throw a PermissionError if the user has the DB open in another program."""
print("Deleting", db_path(name))
close_old_connections()
delete_failed = False
if os.path.exists(db_path(name)): # your database is corrupted and must be destroyed
connections[name].close()
try:
# or you could http://stackoverflow.com/a/24501130/2291495
os.remove(db_path(name))
except PermissionError as err: # must still be holding onto the file lock, clear out contents instead
if not fail_ok:
raise err
else: # I already tried manage.py flush. It doesn't do enough
execute = connections[name].cursor().execute
# raw_sql = "select 'drop table ' || name || ';' from sqlite_master where type = 'table';"
execute("PRAGMA writable_schema = 1;")
execute("delete from sqlite_master where type in ('table', 'index', 'trigger');")
execute("PRAGMA writable_schema = 0;")
execute("VACUUM;")
execute("PRAGMA INTEGRITY_CHECK;")
print("===Dropping all Tables===")
else:
print(db_path(name), "does not exist")
#creates a new blank file by migrate
call_command('migrate', database=name, interactive=False, fake_initial=True)
if name == 'default': # create super user
create_super_user()
def tearDown(self):
from django import db
db.close_old_connections()
for cmd in ("DROP DATABASE %(dbname)s;", "DROP USER %(username)s;"):
complete_cmd = 'sudo -u postgres psql -c "%s"' % (cmd % self.data)
os.system(complete_cmd)
BaseTest.tearDown(self)
def setup_from_none():
if six.PY3:
clear_modules()
from lucterios.framework.settings import fill_appli_settings
import types
import gc
gc.collect()
lct_modules = []
glob = LucteriosGlobal()
_, mod_applis, mod_modules = glob.installed()
for mod_item in mod_applis:
lct_modules.append(mod_item[0])
for mod_item in mod_modules:
lct_modules.append(mod_item[0])
try:
module = types.ModuleType("default_setting")
except TypeError:
module = types.ModuleType(six.binary_type("default_setting"))
setattr(module, '__file__', "")
setattr(module, 'SECRET_KEY', "default_setting")
setattr(
module, 'DATABASES', {'default': {'ENGINE': 'django.db.backends.dummy'}})
fill_appli_settings("lucterios.standard", lct_modules, module)
sys.modules["default_setting"] = module
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "default_setting")
import django
from django import db
django.setup()
db.close_old_connections()
def save_task(task):
"""
Saves the task package to Django or the cache
"""
# SAVE LIMIT < 0 : Don't save success
if not task.get("save", Conf.SAVE_LIMIT > 0) and task["success"]:
return
# SAVE LIMIT > 0: Prune database, SAVE_LIMIT 0: No pruning
db.close_old_connections()
try:
if task["success"] and 0 < Conf.SAVE_LIMIT <= Success.objects.count():
Success.objects.last().delete()
Task.objects.create(
id=task["id"],
name=task["name"],
func=task["func"],
hook=task.get("hook"),
args=task["args"],
kwargs=task["kwargs"],
started=task["started"],
stopped=task["stopped"],
result=task["result"],
group=task.get("group"),
success=task["success"],
)
except Exception as e:
logger.error(e)
def generate_master_hashes():
"""
Generate missing sha1 hashes
"""
# mysql does not support remote/streaming cursors
# to save memory items are loaded from db individually
values = Media.objects.filter(master_sha1__isnull=True).order_by('pk').values('id').nocache()
item_ids = [i['id'] for i in values]
with click.progressbar(item_ids, show_pos=True, width=48,label='Reprocessing {} hashes'.format(len(item_ids))) as bar:
for item_pk in bar:
close_old_connections()
item = Media.objects.get(pk=item_pk)
if item.master and item.master.path:
master_sha1 = item.generate_sha1()
# update media duration
Media.objects.filter(pk=item.pk).update(master_sha1=master_sha1)
def _get_instance_details(instance):
try:
instancedetails.extend(generate_json_light(instance, user))
except (GanetiApiError, Exception):
pass
finally:
close_old_connections()
def worker(bits):
# We need to reset the connections, otherwise the different processes
# will try to share the connection, which causes things to blow up.
from django.db import connections
for alias, info in connections.databases.items():
# We need to also tread lightly with SQLite, because blindly wiping
# out connections (via ``... = {}``) destroys in-memory DBs.
if 'sqlite3' not in info['ENGINE']:
try:
close_old_connections()
if isinstance(connections.thread_local.connections, dict):
del(connections.thread_local.connections[alias])
else:
delattr(connections.thread_local.connections, alias)
except KeyError:
pass
if bits[0] == 'do_update':
func, model, start, end, total, using, start_date, end_date, verbosity, commit = bits
elif bits[0] == 'do_remove':
func, model, pks_seen, start, upper_bound, using, verbosity, commit = bits
else:
return
unified_index = haystack_connections[using].get_unified_index()
index = unified_index.get_index(model)
backend = haystack_connections[using].get_backend()
if func == 'do_update':
qs = index.build_queryset(start_date=start_date, end_date=end_date)
do_update(backend, index, qs, start, end, total, verbosity=verbosity, commit=commit)
else:
raise NotImplementedError('Unknown function %s' % func)
def reload_period_from_analytics(cls, period, verbose=False):
"""Replace the stats for the given period from Google Analytics."""
counts = googleanalytics.pageviews_by_document(
*period_dates(period), verbose=verbose)
if counts:
# Close any existing connections because our load balancer times
# them out at 5 minutes and the GA calls take forever.
close_old_connections()
# Delete and remake the rows:
# Horribly inefficient until
# http://code.djangoproject.com/ticket/9519 is fixed.
# cls.objects.filter(period=period).delete()
# Instead, we use raw SQL!
cursor = connection.cursor()
cursor.execute(
'DELETE FROM `dashboards_wikidocumentvisits`'
' WHERE `period` = %s',
[period])
# Now we create them again with fresh data.
for doc_id, visits in counts.iteritems():
cls.objects.create(document=Document(pk=doc_id), visits=visits,
period=period)
else:
# Don't erase interesting data if there's nothing to replace it:
log.warning('Google Analytics returned no interesting data,'
' so I kept what I had.')
def update_worker(args):
if len(args) != 10:
LOG.error('update_worker received incorrect arguments: %r', args)
raise ValueError('update_worker received incorrect arguments')
model, start, end, total, using, start_date, end_date, verbosity, commit, max_retries = args
# FIXME: confirm that this is still relevant with modern versions of Django:
# We need to reset the connections, otherwise the different processes
# will try to share the connection, which causes things to blow up.
from django.db import connections
for alias, info in connections.databases.items():
# We need to also tread lightly with SQLite, because blindly wiping
# out connections (via ``... = {}``) destroys in-memory DBs.
if 'sqlite3' not in info['ENGINE']:
try:
close_old_connections()
if isinstance(connections.thread_local.connections, dict):
del connections.thread_local.connections[alias]
else:
delattr(connections.thread_local.connections, alias)
except KeyError:
pass
# Request that the connection clear out any transient sessions, file handles, etc.
haystack_connections[using].reset_sessions()
unified_index = haystack_connections[using].get_unified_index()
index = unified_index.get_index(model)
backend = haystack_connections[using].get_backend()
qs = index.build_queryset(start_date=start_date, end_date=end_date)
do_update(backend, index, qs, start, end, total, verbosity, commit, max_retries)
return args
def start():
if os.path.exists(settings.pid_file):
with open(settings.pid_file) as f:
try:
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
fcntl.flock(f, fcntl.LOCK_UN)
except IOError:
sys.stdout.write("zmapd is already started\n")
return
run_daemon_process(pid_file=settings.pid_file, start_msg="Start zmapd(%s)\n")
pid_file = open(settings.pid_file)
fcntl.flock(pid_file, fcntl.LOCK_SH)
while True:
time.sleep(1)
running_jobs = Job.objects.filter(status=Job.STATUS_RUNNING)
total_bandwidth = 0
for job in running_jobs:
total_bandwidth += job.bandwidth
if total_bandwidth >= settings.max_bandwidth:
logger.debug(u"Achieve maximum bandwidth:%sM", settings.max_bandwidth)
continue
jobs = [x for x in Job.objects.filter(status=Job.STATUS_PENDING).order_by('-priority')]
db.close_old_connections()
for j in jobs:
p = multiprocessing.Process(target=execute_job, args=(j.id,))
p.start()
def groups_for_user(environ, username):
"""
Authorizes a user based on groups
"""
# We cannot load these modules before calling get_wsgi_application()
# So they must be defined here.
from ngw.core import perms
from ngw.core.models import ContactGroup
UserModel = auth.get_user_model()
db.reset_queries()
try:
try:
user = UserModel._default_manager.get_by_natural_key(username)
except UserModel.DoesNotExist:
return []
if not user.is_active:
return []
groups = ContactGroup.objects.with_user_perms(
user.id, wanted_flags=perms.VIEW_FILES, add_column=False)
return [force_bytes(group.id) for group in groups]
finally:
db.close_old_connections()
def __init__(self, uuid, receiver_ip, replica, rt=None):
self.uuid = uuid
self.receiver_ip = receiver_ip
self.receiver_port = replica.data_port
self.replica = replica
# TODO: may need to send local shareId so it can be verifed remotely
self.snap_name = '%s_%d_replication' % (replica.share, replica.id)
self.snap_name += '_1' if (rt is None) else '_%d' % (rt.id + 1)
self.snap_id = '%s_%s' % (self.uuid, self.snap_name)
self.rt = rt
self.rt2 = None
self.rt2_id = None
self.rid = replica.id
self.identity = u'%s-%s' % (self.uuid, self.rid)
self.sp = None
# Latest snapshot per Receiver(comes along with receiver-ready)
self.rlatest_snap = None
self.ctx = zmq.Context()
self.msg = ''
self.update_trail = False
self.total_bytes_sent = 0
self.ppid = os.getpid()
self.max_snap_retain = settings.REPLICATION.get('max_snap_retain')
db.close_old_connections()
super(Sender, self).__init__()
def check_password(environ, username, password):
"""
Authenticates against Django's auth database
mod_wsgi docs specify None, True, False as return value depending
on whether the user exists and authenticates.
Manually control this function instead of django
in order to parse environ for correct path
"""
UserModel = auth.get_user_model()
# db connection state is managed similarly to the wsgi handler
# as mod_wsgi may call these functions outside of a request/response cycle
db.reset_queries()
try:
try:
user = UserModel._default_manager.get_by_natural_key(username)
except UserModel.DoesNotExist:
return None
if not user.is_active:
return None
# Method to deny access to specific directories
# if "/tag0.1/scripts/visualisation" in environ['REQUEST_URI']:
# return None
return user.check_password(password)
finally:
db.close_old_connections()
def import_naadsm_scenario(request, new_name=None):
if 'POST' in request.method:
initialized_form = ImportForm(request.POST, request.FILES)
else: # GET page for the first time
initialized_form = ImportForm()
close_old_connections() # close old session
new_scenario(new_name=new_name) # add a scenario to session
context = {'form': initialized_form,
'title': "Import Legacy NAADSM Scenario in XML format",
'loading_message': "Please wait as we import your file...",
'error_title': "Import Error:",
'base_page': 'ScenarioCreator/crispy-model-form.html',
}
if initialized_form.is_valid():
try:
run_importer(request, new_name=new_name)
return loading_screen(request)
except Exception as e:
import sys
info = sys.exc_info()
# stacktrace = info[3]
# print(stacktrace)
# traceback.print_exc()
print(info)
print(str(info[2]))
context['form_errors'] = mark_safe(str(info[1]))
# go on to serve the form normally with Error messages attached
else:
context['form_errors'] = initialized_form.errors
return render(request, 'ScenarioCreator/MainPanel.html', context) # render in validation error messages
def process_image_list(l):
db.close_old_connections()
pool = Pool()
pool.starmap(migrate_image_resize, l)
pool.close()
pool.join()
def django_db_management():
# type: () -> None
reset_queries()
close_old_connections()
try:
yield
finally:
close_old_connections()
def has_user(self, username):
try:
u = User.objects.get(username = username)
db.close_old_connections()
return True
except:
db.close_old_connections()
return False
def setUp(self):
self.client.get('/app/OpenScenario/Roundtrip_test.sqlite3/')
settings = OutputSettings.objects.first()
settings.stop_criteria = 'stop-days'
settings.days = 10
settings.save()
close_old_connections()
def call(*args, **kwargs): try: return fn(*args, **kwargs) finally: if transaction.is_dirty(): transaction.commit() db.connections.all()[0].close() db.close_old_connections()
def call_test_func():
try:
test_func(*args, **kwargs)
except Exception as e:
exceptions.append(e)
raise
finally:
db.close_old_connections()
def worker(task_queue, result_queue, timer, timeout=Conf.TIMEOUT):
"""
Takes a task from the task queue, tries to execute it and puts the result back in the result queue
:type task_queue: multiprocessing.Queue
:type result_queue: multiprocessing.Queue
:type timer: multiprocessing.Value
"""
name = current_process().name
logger.info(_('{} ready for work at {}').format(name, current_process().pid))
task_count = 0
# Start reading the task queue
for task in iter(task_queue.get, 'STOP'):
result = None
timer.value = -1 # Idle
task_count += 1
# Get the function from the task
logger.info(_('{} processing [{}]').format(name, task['name']))
f = task['func']
# if it's not an instance try to get it from the string
if not callable(task['func']):
try:
module, func = f.rsplit('.', 1)
m = importlib.import_module(module)
f = getattr(m, func)
except (ValueError, ImportError, AttributeError) as e:
result = (e, False)
if error_reporter:
error_reporter.report()
if rollbar:
rollbar.report_exc_info()
# We're still going
if not result:
db.close_old_connections()
timer_value = task['kwargs'].pop('timeout', timeout or 0)
# signal execution
pre_execute.send(sender="django_q", func=f, task=task)
# execute the payload
timer.value = timer_value # Busy
try:
res = f(*task['args'], **task['kwargs'])
result = (res, True)
except Exception as e:
result = ('{}'.format(e), False)
if error_reporter:
error_reporter.report()
if rollbar:
rollbar.report_exc_info()
# Process result
task['result'] = result[0]
task['success'] = result[1]
task['stopped'] = timezone.now()
result_queue.put(task)
timer.value = -1 # Idle
# Recycle
if task_count == Conf.RECYCLE:
timer.value = -2 # Recycled
break
logger.info(_('{} stopped doing work').format(name))
def wrapper(*args, **kwargs):
try:
return f(*args, **kwargs)
finally:
close_old_connections()
django.setup()
from django.db import close_old_connections
from imdb.utils.logger import get_root_logger
from imdb.db.settings.pool import init_pool
from imdb.session.interface import MySessionInterface
from imdb.rest.ping import Ping
from imdb.rest.login import Login
from imdb.rest.logout import Logout
from imdb.rest.user import User
from imdb.rest.movie import Movie
logger = get_root_logger()
close_old_connections()
init_pool()
app = Flask("imdb")
CORS(app)
app.auth_header_name = 'Authorization'
app.session_interface = MySessionInterface()
app.root_dir = dirname(dirname(abspath(__file__)))
api = Api(app, prefix='/imdb-service/')
logger.info("Setting up Resources")
api.add_resource(Ping, 'ping/')
api.add_resource(Login, 'login/')
def __call__(self, scope):
self.scope = scope
close_old_connections()
self.intercept()
return self.inner(self.scope)
def ThreadMain(self, ProcName):
logger.info('Starting ' + ProcName)
while 1:
try:
if self.mDieFlag == 1: break # Request for death
self.mLock.acquire()
PauseTime, self.Run = ESSProc.objects.filter(
Name=ProcName).values_list('Time', 'Run')[0]
if self.Run == '0':
logger.info('Stopping ' + ProcName)
ESSProc.objects.filter(Name=ProcName).update(Status='0',
Run='0',
PID=0)
self.RunFlag = 0
self.mLock.release()
if Debug: logger.info('RunFlag: 0')
time.sleep(2)
continue
# Process Item
lock = thread.allocate_lock()
if Debug: logger.info('Start to list worklist')
#######################################
# Start to list robot req
robotQueue_objs = robotQueue.objects.filter(
ReqType__in=[50, 51,
52], Status=0) #Get pending robot requests
for robotQueue_obj in robotQueue_objs:
if ESSProc.objects.get(Name=ProcName).Run == '0':
logger.info('Stopping ' + ProcName)
ESSProc.objects.filter(Name=ProcName).update(
Status='0', Run='0', PID=0)
thread.interrupt_main()
break
t_id = robotQueue_obj.MediumID
if robotQueue_obj.ReqType == 50: #Mount
##########################################
# Check if tape is already mounted
robotdrives_objs = robotdrives.objects.filter(
t_id=t_id, status='Mounted')
if robotdrives_objs:
robotdrives_obj = robotdrives_objs[0]
##########################################
# Tape is mounted, check if locked
if len(robotdrives_obj.drive_lock) > 0:
########################################
# Tape is locked, check if req work_uuid = lock
if robotdrives_obj.drive_lock == robotQueue_obj.ReqUUID:
########################################
# Tape is already locked with req work_uuid
logger.info(
'Already Mounted: ' + str(t_id) +
' and locked by req work_uuid: ' +
str(robotQueue_obj.ReqUUID))
robotQueue_obj.delete()
else:
########################################
# Tape is locked with another work_uuid
logger.info(
'Tape: ' + str(t_id) +
' is busy and locked by: ' +
str(robotdrives_obj.drive_lock) +
' and not req work_uuid: ' +
str(robotQueue_obj.ReqUUID))
else:
########################################
# Tape is not locked, lock the drive with req work_uuid
robotdrives_obj.drive_lock = robotQueue_obj.ReqUUID
robotdrives_obj.save(
update_fields=['drive_lock'])
logger.info(
'Tape: ' + str(t_id) +
' is available set lock to req work_uuid: '
+ str(robotQueue_obj.ReqUUID))
robotQueue_obj.delete()
else:
##########################################
# Tape is not mounted, check for available tape drives
robotdrives_objs = robotdrives.objects.filter(
status='Ready') #Get available tape drives
if robotdrives_objs:
robotdrives_obj = robotdrives_objs[0]
########################################
# Tapedrives is available try to mount tape
robotQueue_obj.Status = 5
robotQueue_obj.save(update_fields=['Status'])
try:
Robot().Mount(t_id,
robotdrives_obj.drive_id,
robotQueue_obj.ReqUUID)
except RobotException as e:
logger.error('Problem to mount tape: ' +
t_id + ' Message: ' + e)
robotQueue_obj.Status = 100
robotQueue_obj.save(
update_fields=['Status'])
else:
#if returncode == 0:
robotQueue_obj.delete()
if storageMedium.objects.filter(
storageMediumID=t_id).exists():
######################################################
# Update StorageMediumTable with num of mounts
storageMedium_obj = storageMedium.objects.get(
storageMediumID=t_id)
timestamp_utc = datetime.datetime.utcnow(
).replace(microsecond=0,
tzinfo=pytz.utc)
timestamp_dst = timestamp_utc.astimezone(
self.tz)
storageMedium_obj.storageMediumMounts += 1
storageMedium_obj.storageMediumLocationStatus = 50
storageMedium_obj.linkingAgentIdentifierValue = AgentIdentifierValue
storageMedium_obj.LocalDBdatetime = timestamp_utc
storageMedium_obj.save(update_fields=[
'storageMediumMounts',
'storageMediumLocationStatus',
'linkingAgentIdentifierValue',
'LocalDBdatetime'
])
if ExtDBupdate:
ext_res, ext_errno, ext_why = ESSMSSQL.DB(
).action(
'storageMedium', 'UPD',
('storageMediumLocationStatus',
50, 'storageMediumMounts',
storageMedium_obj.
storageMediumMounts,
'linkingAgentIdentifierValue',
AgentIdentifierValue),
('storageMediumID', t_id))
if ext_errno:
logger.error(
'Failed to update External DB: '
+ str(t_id) + ' error: ' +
str(ext_why))
else:
storageMedium_obj.ExtDBdatetime = timestamp_utc
storageMedium_obj.save(
update_fields=[
'ExtDBdatetime'
])
#else:
#logger.error('Problem to mount tape: ' + t_id + ' Message: ' + str(mountout))
#robotQueue_obj.Status=100
#robotQueue_obj.save(update_fields=['Status'])
elif robotQueue_obj.ReqType in [
51, 52
]: # Unmount(51), F_Unmount(52)
######################################
# Check if tape is mounted
robotdrives_objs = robotdrives.objects.filter(
t_id=t_id) #Get tape drive that is mounted
if robotdrives_objs:
robotdrives_obj = robotdrives_objs[0]
################################################
# Tape is mounted, check if tape is locked(busy)
if len(robotdrives_obj.drive_lock
) == 0 or robotQueue_obj.ReqType == 52:
if len(robotdrives_obj.drive_lock
) > 0 and robotQueue_obj.ReqType == 52:
logger.info(
'Tape %s is locked with work_uuid %s, try to force unmount',
t_id, robotdrives_obj.drive_lock)
###################################################
# Try to unmount
robotQueue_obj.Status = 5
robotQueue_obj.save(update_fields=['Status'])
robotdrives_obj.status = 'Unmounting'
robotdrives_obj.save(update_fields=['status'])
try:
Robot().Unmount(t_id,
robotdrives_obj.drive_id)
except RobotException as e:
logger.error('Problem to unmount tape: ' +
t_id + ' Message: ' + e)
robotQueue_obj.Status = 100
robotQueue_obj.save(
update_fields=['Status'])
else:
#if returncode == 0:
robotQueue_obj.delete()
#else:
#logger.error('Problem to unmount tape: ' + t_id + ' Message: ' + str(mountout))
#robotQueue_obj.Status=100
#robotQueue_obj.save(update_fields=['Status'])
else:
#################################################
# Tape is locked, skip to unmount
logger.info('Tape ' + t_id +
' is locked, skip to unmount')
else:
################################################
# Tape is not mounted, skip to try to unmount
logger.info('Tape ' + t_id + ' is not mounted')
robotQueue_obj.delete()
db.close_old_connections()
self.mLock.release()
time.sleep(int(PauseTime))
except:
logger.error('Unexpected error: %s' % (str(sys.exc_info())))
raise
self.mDieFlag = 0
def ThreadMain(self, ProcName):
logging.info('Starting ' + ProcName)
TimeZone = timezone.get_default_timezone_name()
self.tz = pytz.timezone(TimeZone)
METS_NAMESPACE = SchemaProfile.objects.get(
entity='mets_namespace').value
METS_SCHEMALOCATION = SchemaProfile.objects.get(
entity='mets_schemalocation').value
METS_PROFILE = SchemaProfile.objects.get(entity='mets_profile').value
XLINK_NAMESPACE = SchemaProfile.objects.get(
entity='xlink_namespace').value
XSI_NAMESPACE = SchemaProfile.objects.get(entity='xsi_namespace').value
while 1:
if self.mDieFlag == 1: break # Request for death
self.mLock.acquire()
self.Time, self.Run = ESSDB.DB().action('ESSProc', 'GET',
('Time', 'Run'),
('Name', ProcName))[0]
if self.Run == '0':
logging.info('Stopping ' + ProcName)
ESSDB.DB().action('ESSProc', 'UPD',
('Status', '0', 'Run', '0', 'PID', '0'),
('Name', ProcName))
self.RunFlag = 0
self.mLock.release()
if Debug: logging.info('RunFlag: 0')
time.sleep(2)
continue
# Process Item
lock = thread.allocate_lock()
Cmets_obj = Parameter.objects.get(
entity='content_descriptionfile').value
self.IngestTable = ESSDB.DB().action('ESSConfig', 'GET',
('Value', ),
('Name', 'IngestTable'))[0][0]
if ExtDBupdate:
self.ext_IngestTable = self.IngestTable
else:
self.ext_IngestTable = ''
self.dbget, errno, why = ESSDB.DB().action(
self.IngestTable, 'GET4',
('ObjectIdentifierValue', 'ObjectUUID', 'PolicyId',
'ObjectSize'), ('StatusProcess', 'BETWEEN', 39, 'AND', 40,
'AND', 'StatusActivity', '=', '0'))
if errno:
logging.error('Failed to access Local DB, error: ' + str(why))
for self.obj in self.dbget:
self.ok = 1
self.ProcDB = ESSDB.DB().action('ESSProc', 'GET',
('Run', 'Pause'),
('Name', ProcName))[0]
if self.ProcDB[0] == '0':
logging.info('Stopping ' + ProcName)
ESSDB.DB().action('ESSProc', 'UPD',
('Status', '0', 'Run', '0', 'PID', '0'),
('Name', ProcName))
thread.interrupt_main()
time.sleep(5)
break
elif self.ProcDB[1] == 1:
while 1:
time.sleep(60)
self.ProcDB = ESSDB.DB().action(
'ESSProc', 'GET', ('Run', 'Pause'),
('Name', ProcName))[0]
if self.ProcDB[1] == 1:
logging.info('Process is in pause state')
else:
break
self.ObjectIdentifierValue = self.obj[0]
self.ObjectUUID = self.obj[1]
self.PolicyId = self.obj[2]
self.ObjectSize = self.obj[3]
ArchivePolicy_obj = ArchivePolicy.objects.get(
PolicyStat=1, PolicyID=self.PolicyId)
if self.ok:
###########################################################
# set variables
self.AIPpath = ArchivePolicy_obj.AIPpath
self.metatype = ArchivePolicy_obj.IngestMetadata
self.ChecksumAlgorithm = ArchivePolicy_obj.ChecksumAlgorithm
self.CA = dict(ChecksumAlgorithm_CHOICES)[
self.ChecksumAlgorithm]
self.SIPpath = ArchivePolicy_obj.IngestPath
self.p_obj = self.ObjectIdentifierValue + '.tar'
self.ObjectPath = os.path.join(self.AIPpath, self.p_obj)
self.SIProotpath = os.path.join(self.SIPpath,
self.ObjectIdentifierValue)
if self.metatype in [4]:
#self.Cmets_obj = '%s/%s_Content_METS.xml' % (self.ObjectIdentifierValue,self.ObjectIdentifierValue)
#self.Cmets_objpath = os.path.join(self.SIPpath,self.Cmets_obj)
#self.Cmets_obj = Cmets_obj.replace('{uuid}',self.ObjectIdentifierValue)
self.Cmets_obj = Cmets_obj.replace(
'{objid}', self.ObjectIdentifierValue)
self.Cmets_objpath = os.path.join(
self.SIProotpath, self.Cmets_obj)
elif self.metatype in [1, 2, 3]:
self.Cmets_obj = '%s_Content_METS.xml' % (
self.ObjectIdentifierValue)
self.Cmets_objpath = os.path.join(
self.AIPpath, self.Cmets_obj)
self.Pmets_obj = '%s_Package_METS.xml' % (
self.ObjectIdentifierValue)
self.Pmets_objpath = os.path.join(self.AIPpath,
self.Pmets_obj)
self.AIC_UUID = None
self.AIC_UUID_rel_ObjectUUIDs = []
if self.ok:
METS_agent_list = []
METS_altRecordID_list = []
if self.metatype == 1:
############################################
# Object have metatype 1 (METS)
self.METS_LABEL = 'ESSArch AIP'
# Get SIP Content METS information
self.METSfilepath = os.path.join(
self.SIPpath,
self.ObjectIdentifierValue + '/metadata/SIP/' +
self.ObjectIdentifierValue + '_Content_METS.xml')
res_info, res_files, res_struct, error, why = ESSMD.getMETSFileList(
FILENAME=self.METSfilepath)
for agent in res_info[2]:
if not (agent[0] == 'CREATOR'
and agent[3] == 'SOFTWARE'):
METS_agent_list.append(agent)
METS_agent_list.append([
'CREATOR', 'INDIVIDUAL', '', AgentIdentifierValue,
[]
])
METS_agent_list.append([
'CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch',
['VERSION=%s' % ProcVersion]
])
elif self.metatype == 2:
############################################
# Object have metatype 2 (RES)
self.METS_LABEL = 'Imaging AIP RA'
METS_agent_list.append([
'ARCHIVIST', 'ORGANIZATION', '', 'Riksarkivet', []
])
METS_agent_list.append(
['CREATOR', 'ORGANIZATION', '', 'Riksarkivet', []])
METS_agent_list.append([
'CREATOR', 'INDIVIDUAL', '', AgentIdentifierValue,
[]
])
METS_agent_list.append([
'CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch',
['VERSION=%s' % ProcVersion]
])
elif self.metatype == 3:
############################################
# Object have metatype 3 (ADDML)
self.METS_LABEL = 'Born Digital AIP RA'
METS_agent_list.append([
'ARCHIVIST', 'ORGANIZATION', '', 'Riksarkivet', []
])
METS_agent_list.append(
['CREATOR', 'ORGANIZATION', '', 'Riksarkivet', []])
METS_agent_list.append([
'CREATOR', 'INDIVIDUAL', '', AgentIdentifierValue,
[]
])
METS_agent_list.append([
'CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch',
['VERSION=%s' % ProcVersion]
])
elif self.metatype in [4]:
############################################
# Object have metatype 4 (eARD METS)
res_info, res_files, res_struct, error, why = ESSMD.getMETSFileList(
FILENAME=self.Cmets_objpath)
for agent in res_info[2]:
#if not (agent[0] == 'CREATOR' and agent[3] == 'SOFTWARE'):
METS_agent_list.append(agent)
self.METS_LABEL = res_info[0][0]
METS_agent_list.append([
'CREATOR', None, 'INDIVIDUAL', None,
AgentIdentifierValue, []
])
METS_agent_list.append([
'CREATOR', None, 'OTHER', 'SOFTWARE', 'ESSArch',
['VERSION=%s' % ProcVersion]
])
for altRecordID in res_info[3]:
METS_altRecordID_list.append(altRecordID)
logging.debug('self.obj: ' + str(self.obj))
if self.ChecksumAlgorithm > 0: #self.ChecksumAlgorithm 1 = MD5, 2 = SHA-256
self.startCalTime = datetime.timedelta(
seconds=time.localtime()[5],
minutes=time.localtime()[4],
hours=time.localtime()[3])
errno, why = ESSPGM.DB().SetAIPstatus(
self.IngestTable, self.ext_IngestTable,
AgentIdentifierValue, self.ObjectUUID, 40, 5)
if errno:
logging.error('Failed to update DB status for AIP: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
logging.info('Start create Package METS for: ' +
self.ObjectIdentifierValue)
if self.ok:
###########################################################
# Create PMETS for AIP package
self.M_CHECKSUM, errno, why = ESSPGM.Check().checksum(
self.Cmets_objpath, self.CA)
if errno:
self.event_info = 'Problem to get checksum for METS object for AIP package: ' + str(
self.Cmets_objpath)
logging.error(self.event_info)
ESSPGM.Events().create('1030', '',
'ESSArch AIPCreator',
ProcVersion, '1',
self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
self.M_statinfo = os.stat(self.Cmets_objpath)
self.M_SIZE = self.M_statinfo.st_size
self.M_utc_mtime = datetime.datetime.utcfromtimestamp(
self.M_statinfo.st_mtime).replace(tzinfo=pytz.utc)
self.M_lociso_mtime = self.M_utc_mtime.astimezone(
self.tz).isoformat()
self.P_CHECKSUM, errno, why = ESSPGM.Check().checksum(
self.ObjectPath, self.CA)
if errno:
self.event_info = 'Problem to get checksum for AIP package: ' + str(
self.ObjectPath)
logging.error(self.event_info)
ESSPGM.Events().create('1040', '',
'ESSArch AIPChecksum',
ProcVersion, '1',
self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
self.P_statinfo = os.stat(self.ObjectPath)
self.P_SIZE = self.P_statinfo.st_size
self.P_utc_mtime = datetime.datetime.utcfromtimestamp(
self.P_statinfo.st_mtime).replace(tzinfo=pytz.utc)
self.P_lociso_mtime = self.P_utc_mtime.astimezone(
self.tz).isoformat()
if self.metatype in [1, 2, 3]:
self.PMETSdoc = ESSMD.createPMets(
ID=self.ObjectIdentifierValue,
LABEL=self.METS_LABEL,
AGENT=METS_agent_list,
P_SIZE=self.P_SIZE,
P_CREATED=self.P_lociso_mtime,
P_CHECKSUM=self.P_CHECKSUM,
P_CHECKSUMTYPE=self.CA,
M_SIZE=self.M_SIZE,
M_CREATED=self.M_lociso_mtime,
M_CHECKSUM=self.M_CHECKSUM,
M_CHECKSUMTYPE=self.CA,
)
errno, why = ESSMD.writeToFile(
self.PMETSdoc, self.Pmets_objpath)
if errno:
self.event_info = 'Problem to write PMETS to file for AIP package: ' + str(
self.Pmets_objpath)
logging.error(self.event_info)
ESSPGM.Events().create(
'1040', '', 'ESSArch AIPChecksum',
ProcVersion, '1', self.event_info, 2,
self.ObjectIdentifierValue)
time.sleep(2)
self.ok = 0
elif self.metatype in [4]:
ms_files = []
ms_files.append([
'amdSec', None, 'techMD', 'techMD001', None,
None,
'ID%s' % str(uuid.uuid1()), 'URL',
'file:%s/%s' %
(self.ObjectIdentifierValue, self.Cmets_obj),
'simple', self.M_CHECKSUM, self.CA,
self.M_SIZE, 'text/xml', self.M_lociso_mtime,
'OTHER', 'METS', None
])
ms_files.append([
'fileSec', None, None, None, None, None,
'ID%s' % str(uuid.uuid1()), 'URL',
'file:%s' % self.p_obj, 'simple',
self.P_CHECKSUM, self.CA, self.P_SIZE,
'application/x-tar', self.P_lociso_mtime,
'tar', 'techMD001', None
])
# define namespaces
self.namespacedef = 'xmlns:mets="%s"' % METS_NAMESPACE
self.namespacedef += ' xmlns:xlink="%s"' % XLINK_NAMESPACE
self.namespacedef += ' xmlns:xsi="%s"' % XSI_NAMESPACE
self.namespacedef += ' xsi:schemaLocation="%s %s"' % (
METS_NAMESPACE, METS_SCHEMALOCATION)
errno, info_list = ESSMD.Create_IP_mets(
ObjectIdentifierValue=self.
ObjectIdentifierValue,
METS_ObjectPath=self.Pmets_objpath,
agent_list=METS_agent_list,
altRecordID_list=METS_altRecordID_list,
file_list=ms_files,
namespacedef=self.namespacedef,
METS_LABEL=self.METS_LABEL,
METS_PROFILE=METS_PROFILE,
METS_TYPE='AIP',
METS_DocumentID=self.Pmets_obj,
TimeZone=TimeZone)
if errno:
logging.error(
'Problem to create Package METS file, why: %s'
% str(info_list))
self.ObjectMessageDigest = self.P_CHECKSUM
self.stopCalTime = datetime.timedelta(
seconds=time.localtime()[5],
minutes=time.localtime()[4],
hours=time.localtime()[3])
self.CalTime = self.stopCalTime - self.startCalTime
self.ObjectSizeMB = self.ObjectSize / 1048576
if self.CalTime.seconds < 1:
self.CalTime = datetime.timedelta(
seconds=1
) #Fix min time to 1 second if it is zero.
self.CalMBperSEC = int(self.ObjectSizeMB) / int(
self.CalTime.seconds)
logging.info('Finished calculate checksum: ' +
self.ObjectIdentifierValue + ' , ' +
str(self.CalMBperSEC) + ' MB/Sec and Time: ' +
str(self.CalTime))
if self.ok:
self.timestamp_utc = datetime.datetime.utcnow(
).replace(microsecond=0, tzinfo=pytz.utc)
self.timestamp_dst = self.timestamp_utc.astimezone(
self.tz)
res, errno, why = ESSDB.DB().action(
self.IngestTable, 'UPD',
('ObjectMessageDigestAlgorithm',
self.ChecksumAlgorithm, 'ObjectMessageDigest',
self.ObjectMessageDigest, 'MetaObjectSize',
self.M_SIZE, 'LastEventDate',
self.timestamp_utc.replace(tzinfo=None),
'linkingAgentIdentifierValue',
AgentIdentifierValue, 'LocalDBdatetime',
self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if errno:
logging.error('Failed to update Local DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
if errno == 0 and ExtDBupdate:
ext_res, ext_errno, ext_why = ESSMSSQL.DB().action(
self.IngestTable, 'UPD',
('ObjectMessageDigestAlgorithm',
self.ChecksumAlgorithm, 'ObjectMessageDigest',
self.ObjectMessageDigest, 'MetaObjectSize',
self.M_SIZE, 'LastEventDate',
self.timestamp_dst.replace(tzinfo=None),
'linkingAgentIdentifierValue',
AgentIdentifierValue),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if ext_errno:
logging.error(
'Failed to update External DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(ext_why))
else:
res, errno, why = ESSDB.DB().action(
self.IngestTable, 'UPD',
('ExtDBdatetime',
self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if errno:
logging.error(
'Failed to update Local DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
if self.ok and self.metatype == 4:
####################################################
# Create AIC METS File:
aic_obj = ArchiveObject.objects.filter(
relaic_set__UUID=self.ObjectUUID)[:1]
if aic_obj:
self.AIC_UUID = aic_obj.get().ObjectUUID
logging.info(
'Succeeded to get AIC_UUID: %s from DB' %
self.AIC_UUID)
else:
logging.warning(
'AIC not found for IP object: %s, skip to create AIC METS file'
% self.ObjectUUID)
if self.ok and self.AIC_UUID:
ip_obj_list = ArchiveObject.objects.filter(
Q(StatusProcess=3000)
| Q(ObjectUUID=self.ObjectUUID),
reluuid_set__AIC_UUID=self.AIC_UUID).order_by(
'Generation')
if ip_obj_list:
logging.info('Start create AIC METS: ' +
self.AIC_UUID)
self.AICmets_objpath = os.path.join(
self.AIPpath, self.AIC_UUID + '_AIC_METS.xml')
ms_files = []
for ip_obj in ip_obj_list:
logging.info(
'Add IP: %s to AIC METS: %s' %
(ip_obj.ObjectUUID, self.AIC_UUID))
ms_files.append([
'fileSec', None, None, None, None, None,
'ID%s' % str(uuid.uuid1()), 'URL',
'file:%s' % ip_obj.ObjectUUID, 'simple',
ip_obj.ObjectMessageDigest,
dict(ChecksumAlgorithm_CHOICES)[
ip_obj.ObjectMessageDigestAlgorithm],
ip_obj.ObjectSize, 'application/x-tar',
ip_obj.CreateDate, 'IP Package', None, None
])
# define namespaces
self.namespacedef = 'xmlns:mets="%s"' % METS_NAMESPACE
self.namespacedef += ' xmlns:xlink="%s"' % XLINK_NAMESPACE
self.namespacedef += ' xmlns:xsi="%s"' % XSI_NAMESPACE
self.namespacedef += ' xsi:schemaLocation="%s %s"' % (
METS_NAMESPACE, METS_SCHEMALOCATION)
errno, info_list = ESSMD.Create_IP_mets(
ObjectIdentifierValue=self.AIC_UUID,
METS_ObjectPath=self.AICmets_objpath,
agent_list=[],
altRecordID_list=[],
file_list=ms_files,
namespacedef=self.namespacedef,
METS_LABEL='AIC relation to IP',
METS_PROFILE=METS_PROFILE,
METS_TYPE='AIC',
METS_DocumentID=self.AIC_UUID +
'_AIC_METS.xml',
TimeZone=TimeZone)
if errno:
logging.error(
'Problem to create AIC METS file, why: %s'
% str(info_list))
else:
logging.error(
'Problem to get objects related to AIC_UUID: %s from DB'
% (self.AIC_UUID))
self.ok = 0
if self.ok:
errno, why = ESSPGM.DB().SetAIPstatus(
self.IngestTable, self.ext_IngestTable,
AgentIdentifierValue, self.ObjectUUID, 49, 0)
if errno:
logging.error(
'Failed to update DB status for AIP: ' +
str(self.ObjectIdentifierValue) + ' error: ' +
str(why))
else:
self.event_info = 'Succeeded to create checksum for Object: %s' % self.ObjectIdentifierValue
logging.info(self.event_info)
ESSPGM.Events().create('1040', '',
'ESSArch AIPChecksum',
ProcVersion, '0',
self.event_info, 2,
self.ObjectIdentifierValue)
else:
errno, why = ESSPGM.DB().SetAIPstatus(
self.IngestTable, self.ext_IngestTable,
AgentIdentifierValue, self.ObjectUUID, 40, 100)
if errno:
logging.error(
'Failed to update DB status for AIP: ' +
str(self.ObjectIdentifierValue) + ' error: ' +
str(why))
else:
self.event_info = 'Failed to create checksum for Object: %s' % self.ObjectIdentifierValue
logging.error(self.event_info)
ESSPGM.Events().create('1040', '',
'ESSArch AIPChecksum',
ProcVersion, '1',
self.event_info, 2,
self.ObjectIdentifierValue)
elif self.ChecksumAlgorithm == 0: #self.ChecksumAlgorithm 0 = No checksum
logging.info('Skip creation of checksum: ' +
self.ObjectIdentifierValue)
self.ObjectMessageDigest = ''
self.MetaObjectSize = os.stat(self.Cmets_objpath)[6]
self.timestamp_utc = datetime.datetime.utcnow().replace(
microsecond=0, tzinfo=pytz.utc)
self.timestamp_dst = self.timestamp_utc.astimezone(self.tz)
res, errno, why = ESSDB.DB().action(
self.IngestTable, 'UPD',
('ObjectMessageDigestAlgorithm',
self.ChecksumAlgorithm, 'ObjectMessageDigest',
self.ObjectMessageDigest, 'StatusProcess', '49',
'StatusActivity', '0', 'MetaObjectSize',
self.MetaObjectSize, 'LastEventDate',
self.timestamp_utc.replace(tzinfo=None),
'linkingAgentIdentifierValue', AgentIdentifierValue,
'LocalDBdatetime',
self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue', self.ObjectIdentifierValue))
if errno:
logging.error('Failed to update Local DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
else:
ESSPGM.Events().create('1040', '',
'ESSArch AIPChecksum',
ProcVersion, '0',
'Skip creation of checksum', 2,
self.ObjectIdentifierValue)
if errno == 0 and ExtDBupdate:
ext_res, ext_errno, ext_why = ESSMSSQL.DB().action(
self.IngestTable, 'UPD',
('ObjectMessageDigestAlgorithm',
self.ChecksumAlgorithm, 'ObjectMessageDigest',
self.ObjectMessageDigest, 'StatusProcess', '49',
'StatusActivity', '0', 'MetaObjectSize',
self.MetaObjectSize, 'LastEventDate',
self.timestamp_dst.replace(tzinfo=None),
'linkingAgentIdentifierValue',
AgentIdentifierValue),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if ext_errno:
logging.error('Failed to update External DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(ext_why))
else:
res, errno, why = ESSDB.DB().action(
self.IngestTable, 'UPD',
('ExtDBdatetime',
self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if errno:
logging.error('Failed to update Local DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
db.close_old_connections()
self.mLock.release()
time.sleep(int(self.Time))
self.mDieFlag = 0
def __init__(self, debug=False, verbosity=1):
"""
Spawn greenlets for handling websockets and PostgreSQL calls.
Only one pgworker should be started for any process, and ideally only
one instance per node as well (ie: use IPC to communicate events between
processes on the same host). This class handles the former (per
process), per host IPC is an excercise left to you, dear reader - pull
requests are of course welcome. ;)
For the record, there's no technical reason that you can't have multiple
pgworker instances running on one host (or even one process), it's just
inefficient to send exactly the same traffic multiple times between your
web servers and your database host.
"""
import logging
from django.apps import apps
from django.utils.module_loading import import_string
from dddp.websocket import DDPWebSocketApplication
self.verbosity = verbosity
self._stop_event = gevent.event.Event()
self._stop_event.set() # start in stopped state.
self.logger = logging.getLogger('dddp.launcher')
if DDPLauncher.pgworker is None:
from django.db import connection, close_old_connections
from dddp.postgres import PostgresGreenlet
# shutdown existing connections
close_old_connections()
DDPLauncher.pgworker = PostgresGreenlet(connection)
# use settings.WSGI_APPLICATION or fallback to default Django WSGI app
from django.conf import settings
self.wsgi_app = None
if hasattr(settings, 'WSGI_APPLICATION'):
self.wsgi_name = settings.WSGI_APPLICATION
try:
self.wsgi_app = import_string(self.wsgi_name)
except ImportError:
pass
if self.wsgi_app is None:
from django.core.wsgi import get_wsgi_application
self.wsgi_app = get_wsgi_application()
self.wsgi_name = str(self.wsgi_app.__class__)
self.api = apps.get_app_config('dddp').api
self.api.pgworker = DDPLauncher.pgworker
DDPWebSocketApplication.api = self.api
# setup PostgresGreenlet to multiplex DB calls
DDPWebSocketApplication.pgworker = self.pgworker
self.resource = geventwebsocket.Resource(
collections.OrderedDict([
(r'/websocket', DDPWebSocketApplication),
(r'^/sockjs/\d+/\w+/websocket$', DDPWebSocketApplication),
(r'^/sockjs/\d+/\w+/xhr$', ddpp_sockjs_xhr),
(r'^/sockjs/info$', ddpp_sockjs_info),
(r'^/(?!(websocket|sockjs)/)', self.wsgi_app),
]), )
self.servers = []
self.threads = []
def wrapped_f(*args, **kwargs):
try:
start = datetime.now()
output = f(*args, **kwargs)
logger.info("Time taken for %s is %s", self.api_key,
datetime.now() - start)
return output
except KeyError as e:
exception_trace = traceback.format_exc()
parsed_request = parse_request_for_error(request)
logger.error("Exception in %s API:\n%s Request: \n%s",
self.api_key, exception_trace, parsed_request)
logger.error("Missing key : " + e.__repr__())
return error_response(400,
"{} is required".format(e.__repr__()))
except (ValidationError, TypeError) as e:
exception_trace = traceback.format_exc()
parsed_request = parse_request_for_error(request)
logger.error("Exception in %s API:\n%s Request: \n%s",
self.api_key, exception_trace, parsed_request)
logger.error("Missing key : " + e.__repr__())
return error_response(
400,
"Bad Request object. Missing Parameters or wrong endpoint")
except ValueError as e:
exception_trace = traceback.format_exc()
parsed_request = parse_request_for_error(request)
logger.error("Exception in %s API:\n%s Request: \n%s",
self.api_key, exception_trace, parsed_request)
logger.error("Missing key : " + e.__repr__())
return error_response(
400, "Bad Request. Incorrect Parameter types")
except (ObjectDoesNotExist, MultipleObjectsReturned) as e:
exception_trace = traceback.format_exc()
parsed_request = parse_request_for_error(request)
logger.error("Exception in %s API:\n%s Request: \n%s",
self.api_key, exception_trace, parsed_request)
logger.error("Missing key : " + e.__repr__())
return error_response(
422,
"Unprocessable Entity, no records found or multiple records found"
)
except PermissionDenied as e:
exception_trace = traceback.format_exc()
parsed_request = parse_request_for_error(request)
logger.error("Exception in %s API:\n%s Request: \n%s",
self.api_key, exception_trace, parsed_request)
logger.error("Missing key : " + e.__repr__())
return error_response(403, "Permission Denied")
except CustomError as error:
exception_trace = traceback.format_exc()
parsed_request = parse_request_for_error(request)
logger.error("Exception in %s API: %s\n%s \nRequest: %s",
self.api_key, error.get_message(),
exception_trace, parsed_request)
return error_response(error.get_error_code(),
error.get_message())
except IntegrityError as e:
exception_trace = traceback.format_exc()
parsed_request = parse_request_for_error(request)
logger.error("Exception in %s API:\n%s Request: \n%s",
self.api_key, exception_trace, parsed_request)
db_err_code, _ = e.args
if db_err_code == 1062:
return error_response(400, "Duplicate Entry")
return error_response(500, "Internal Server Error")
except Exception as e:
exception_trace = traceback.format_exc()
parsed_request = parse_request_for_error(request)
logger.error("Exception in %s API:\n%s Request: \n%s",
self.api_key, exception_trace, parsed_request)
logger.error("Missing key : " + e.__repr__())
return error_response(500, "Internal Server Error")
finally:
close_old_connections()
def new_game(request):
user = request.user
hosted = Game.objects.create(white_player=user)
close_old_connections()
return HttpResponseRedirect(reverse('game', args=[hosted.label]))
def solo_game(request):
user = request.user
solo = Game.objects.create(white_player=user, black_player=user)
close_old_connections()
return HttpResponseRedirect(reverse('game', args=[solo.label]))
def thread_handler(self, loop, *args, **kwargs):
try:
return super().thread_handler(loop, *args, **kwargs)
finally:
close_old_connections()
def handle_creation(job):
global logger
data = json.loads(job.body)
try:
application = InstanceApplication.objects.get(
id=data["application_id"])
except ObjectDoesNotExist:
logger.warn("Unable to find application #%d, burying" %
data["application_id"])
try_log(
mail_admins, "Burying job #%d" % job.jid,
"Please inspect job #%d (application %d) manually" %
(job.jid, data["application_id"]))
job.bury()
close_old_connections()
return
finally:
close_old_connections()
logger.info("Handling %s (job: %d)", application.hostname,
application.job_id)
while True:
sleep(15)
logger.info("Checking %s (job: %d)", application.hostname,
application.job_id)
status = application.cluster.get_job_status(application.job_id)
if status["end_ts"]:
logger.info("%s (job: %d) done. Status: %s", application.hostname,
application.job_id, status["status"])
if status["status"] == "error":
application.status = STATUS_FAILED
application.backend_message = smart_str(status["opresult"])
application.save()
logger.warn("%s (job: %d) failed. Notifying admins",
application.hostname, application.job_id)
try_log(
mail_admins, "Instance creation failure for %s on %s" %
(application.hostname, application.cluster),
json.dumps(status, indent=2))
else:
application.status = STATUS_SUCCESS
application.backend_message = None
application.save()
logger.info("Mailing %s about %s", application.applicant.email,
application.hostname)
fqdn = Site.objects.get_current().domain
instance_url = "https://%s%s" % \
(fqdn, urlresolvers.reverse("instance-detail",
args=(application.cluster.slug,
application.hostname)))
mail_body = render_to_string(
"instances/emails/instance_created_mail.txt", {
"application": application,
"instance_url": instance_url,
"BRANDING": settings.BRANDING
})
mail_body_managers = render_to_string(
"instances/emails/instance_created_mail.txt", {
"application": application,
"reviewer": application.reviewer,
"instance_url": instance_url,
"BRANDING": settings.BRANDING
})
try_log(
send_mail, settings.EMAIL_SUBJECT_PREFIX +
"Instance %s is ready" % application.hostname, mail_body,
settings.SERVER_EMAIL, [application.applicant.email])
logger.info("Mailing managers about %s" % application.hostname)
try_log(mail_managers,
"Instance %s is ready" % application.hostname,
mail_body_managers)
job.delete()
close_old_connections()
break
job.touch()
def run_cronjob_in_thread(self, logs_count):
call(self.wait_3sec_cron)
self.assertEqual(CronJobLog.objects.all().count(), logs_count + 1)
db.close_old_connections()
def inner(*args, **kwargs):
try:
return fn(*args, **kwargs)
finally:
if not HUEY.always_eager:
close_old_connections()
def scheduler(broker=None):
"""
Creates a task from a schedule at the scheduled time and schedules next run
"""
if not broker:
broker = get_broker()
db.close_old_connections()
try:
with db.transaction.atomic():
for s in Schedule.objects.select_for_update().exclude(
repeats=0).filter(next_run__lt=timezone.now()):
args = ()
kwargs = {}
# get args, kwargs and hook
if s.kwargs:
try:
# eval should be safe here because dict()
kwargs = eval('dict({})'.format(s.kwargs))
except SyntaxError:
kwargs = {}
if s.args:
args = ast.literal_eval(s.args)
# single value won't eval to tuple, so:
if type(args) != tuple:
args = (args, )
q_options = kwargs.get('q_options', {})
if s.hook:
q_options['hook'] = s.hook
# set up the next run time
if not s.schedule_type == s.ONCE:
next_run = arrow.get(s.next_run)
while True:
if s.schedule_type == s.MINUTES:
next_run = next_run.replace(
minutes=+(s.minutes or 1))
elif s.schedule_type == s.HOURLY:
next_run = next_run.replace(hours=+1)
elif s.schedule_type == s.DAILY:
next_run = next_run.replace(days=+1)
elif s.schedule_type == s.WEEKLY:
next_run = next_run.replace(weeks=+1)
elif s.schedule_type == s.MONTHLY:
next_run = next_run.replace(months=+1)
elif s.schedule_type == s.QUARTERLY:
next_run = next_run.replace(months=+3)
elif s.schedule_type == s.YEARLY:
next_run = next_run.replace(years=+1)
if Conf.CATCH_UP or next_run > arrow.utcnow():
break
s.next_run = next_run.datetime
s.repeats += -1
# send it to the cluster
q_options['broker'] = broker
q_options['group'] = q_options.get('group', s.name or s.id)
kwargs['q_options'] = q_options
s.task = django_q.tasks.async_task(s.func, *args, **kwargs)
# log it
if not s.task:
logger.error(
_('{} failed to create a task from schedule [{}]').
format(current_process().name, s.name or s.id))
else:
logger.info(
_('{} created a task from schedule [{}]').format(
current_process().name, s.name or s.id))
# default behavior is to delete a ONCE schedule
if s.schedule_type == s.ONCE:
if s.repeats < 0:
s.delete()
continue
# but not if it has a positive repeats
s.repeats = 0
# save the schedule
s.save()
except Exception as e:
logger.error(e)
def inner(*args, **kwargs):
try:
return fn(*args, **kwargs)
finally:
if not HUEY.immediate and not getattr(HUEY, 'testing', False):
close_old_connections()
def foo(*args, **kwargs):
try:
return fn(*args, **kwargs)
finally:
close_old_connections()
def _close_old_connections(self, *args, **kwargs):
db.close_old_connections()
def query(request):
"""
获取SQL查询结果
:param request:
:return:
"""
instance_name = request.POST.get('instance_name')
sql_content = request.POST.get('sql_content')
db_name = request.POST.get('db_name')
tb_name = request.POST.get('tb_name')
limit_num = int(request.POST.get('limit_num', 0))
schema_name = request.POST.get('schema_name', None)
user = request.user
result = {'status': 0, 'msg': 'ok', 'data': {}}
try:
instance = user_instances(
request.user).get(instance_name=instance_name)
except Instance.DoesNotExist:
result['status'] = 1
result['msg'] = '你所在组未关联该实例'
return HttpResponse(json.dumps(result),
content_type='application/json')
# 服务器端参数验证
if None in [sql_content, db_name, instance_name, limit_num]:
result['status'] = 1
result['msg'] = '页面提交参数可能为空'
return HttpResponse(json.dumps(result),
content_type='application/json')
try:
config = SysConfig()
# 查询前的检查，禁用语句检查，语句切分
query_engine = get_engine(instance=instance)
query_check_info = query_engine.query_check(db_name=db_name,
sql=sql_content)
if query_check_info.get('bad_query'):
# 引擎内部判断为 bad_query
result['status'] = 1
result['msg'] = query_check_info.get('msg')
return HttpResponse(json.dumps(result),
content_type='application/json')
if query_check_info.get(
'has_star') and config.get('disable_star') is True:
# 引擎内部判断为有 * 且禁止 * 选项打开
result['status'] = 1
result['msg'] = query_check_info.get('msg')
return HttpResponse(json.dumps(result),
content_type='application/json')
sql_content = query_check_info['filtered_sql']
# 查询权限校验，并且获取limit_num
priv_check_info = query_priv_check(user, instance, db_name,
sql_content, limit_num)
if priv_check_info['status'] == 0:
limit_num = priv_check_info['data']['limit_num']
priv_check = priv_check_info['data']['priv_check']
else:
result['status'] = 1
result['msg'] = priv_check_info['msg']
return HttpResponse(json.dumps(result),
content_type='application/json')
# explain的limit_num设置为0
limit_num = 0 if re.match(r"^explain",
sql_content.lower()) else limit_num
# 对查询sql增加limit限制或者改写语句
sql_content = query_engine.filter_sql(sql=sql_content,
limit_num=limit_num)
# 先获取查询连接，用于后面查询复用连接以及终止会话
query_engine.get_connection(db_name=db_name)
thread_id = query_engine.thread_id
max_execution_time = int(config.get('max_execution_time', 60))
# 执行查询语句，并增加一个定时终止语句的schedule，timeout=max_execution_time
if thread_id:
schedule_name = f'query-{time.time()}'
run_date = (datetime.datetime.now() +
datetime.timedelta(seconds=max_execution_time))
add_kill_conn_schedule(schedule_name, run_date, instance.id,
thread_id)
with FuncTimer() as t:
# 获取主从延迟信息
seconds_behind_master = query_engine.seconds_behind_master
query_result = query_engine.query(db_name,
sql_content,
limit_num,
schema_name=schema_name,
tb_name=tb_name)
query_result.query_time = t.cost
# 返回查询结果后删除schedule
if thread_id:
del_schedule(schedule_name)
# 查询异常
if query_result.error:
result['status'] = 1
result['msg'] = query_result.error
# 数据脱敏，仅对查询无错误的结果集进行脱敏，并且按照query_check配置是否返回
elif config.get('data_masking'):
try:
with FuncTimer() as t:
masking_result = query_engine.query_masking(
db_name, sql_content, query_result)
masking_result.mask_time = t.cost
# 脱敏出错
if masking_result.error:
# 开启query_check，直接返回异常，禁止执行
if config.get('query_check'):
result['status'] = 1
result['msg'] = f'数据脱敏异常：{masking_result.error}'
# 关闭query_check，忽略错误信息，返回未脱敏数据，权限校验标记为跳过
else:
logger.warning(
f'数据脱敏异常，按照配置放行，查询语句：{sql_content}，错误信息：{masking_result.error}'
)
query_result.error = None
result['data'] = query_result.__dict__
# 正常脱敏
else:
result['data'] = masking_result.__dict__
except Exception as msg:
# 抛出未定义异常，并且开启query_check，直接返回异常，禁止执行
if config.get('query_check'):
result['status'] = 1
result['msg'] = f'数据脱敏异常，请联系管理员，错误信息：{msg}'
# 关闭query_check，忽略错误信息，返回未脱敏数据，权限校验标记为跳过
else:
logger.warning(
f'数据脱敏异常，按照配置放行，查询语句：{sql_content}，错误信息：{msg}')
query_result.error = None
result['data'] = query_result.__dict__
# 无需脱敏的语句
else:
result['data'] = query_result.__dict__
# 仅将成功的查询语句记录存入数据库
if not query_result.error:
result['data']['seconds_behind_master'] = seconds_behind_master
if int(limit_num) == 0:
limit_num = int(query_result.affected_rows)
else:
limit_num = min(int(limit_num),
int(query_result.affected_rows))
query_log = QueryLog(username=user.username,
user_display=user.display,
db_name=db_name,
instance_name=instance.instance_name,
sqllog=sql_content,
effect_row=limit_num,
cost_time=query_result.query_time,
priv_check=priv_check,
hit_rule=query_result.mask_rule_hit,
masking=query_result.is_masked)
# 防止查询超时
if connection.connection and not connection.is_usable():
close_old_connections()
query_log.save()
except Exception as e:
logger.error(
f'查询异常报错，查询语句：{sql_content}\n，错误信息：{traceback.format_exc()}')
result['status'] = 1
result['msg'] = f'查询异常报错，错误信息：{e}'
return HttpResponse(json.dumps(result),
content_type='application/json')
# 返回查询结果
try:
return HttpResponse(json.dumps(result,
cls=ExtendJSONEncoderFTime,
bigint_as_string=True),
content_type='application/json')
# 虽然能正常返回，但是依然会乱码
except UnicodeDecodeError:
return HttpResponse(json.dumps(result,
default=str,
bigint_as_string=True,
encoding='latin1'),
content_type='application/json')
def wrapper(*args, **kwargs):
close_old_connections()
return func(*args, **kwargs)
def clear_cluster_users_cache(cluster_slug):
for user in User.objects.all():
cache.delete("user:%s:index:instances" % user.username)
cache.delete("cluster:%s:instances" % cluster_slug)
close_old_connections()
def run_bkgd_search():
"""
Runs background queries.
"""
execute_filter_queries()
close_old_connections()
def import_pgdb_file(tree, user):
global logs
global done
logs = []
done = False
root = tree.getroot()
# all students
for s in root[0]:
try:
if len(Student.objects.filter(student_num=int(s[0].text))) != 0:
print(f"Student with number {s[0].text} already exists")
logs.append(f"Student with number {s[0].text} \t ({s[4].text}, {s[3].text}) already exists")
continue
s_obj = Student(
student_num=int(s[0].text),
cur_grade_num=int(s[1].text),
homeroom_str=f"{s[2].text}",
first=s[3].text.strip(),
last=s[4].text.strip(),
legal=s[5].text.strip(),
sex=s[6].text.strip(),
grad_year=int(s[7].text),
active=(True if s[8].text == "yes" else False),
)
s_obj.save(user)
for g in s[9]:
g_obj = s_obj.get_grade(int(g[0].text))
g_obj.anecdote = g[2].text or ""
g_obj.set_term1_avg(float(g[3].text), user)
g_obj.set_term2_avg(float(g[4].text), user)
for p in g[5]:
if (len(PointCodes.objects.filter(catagory=p[0].text).filter(
code=int(p[1].text))) == 0):
type = PointCodes(catagory=p[0].text, code=int(p[1].text),
description=str(p[0].text) + str(p[1].text))
type.save()
else:
type = PointCodes.objects.filter(catagory=p[0].text).get(code=int(p[1].text))
g_obj.add_point(Points(type=type, amount=float(p[2].text)), user)
g_obj.calc_points_total("SE")
g_obj.calc_points_total("AT")
g_obj.calc_points_total("FA")
g_obj.save()
logs.append(f"Added student {s[0].text} \t ({s[4].text}, {s[3].text}) successfully")
except Exception as e:
raise e
student_num = int(s[0].text)
print(f"Failed to add student {int(s[0].text)}")
logs.append(f"Failed to add student {s[0].text} \t ({s[4].text}, {s[3].text}) {e}")
# delete the partially formed student
if len(Student.objects.filter(student_num=student_num)) != 0:
Student.objects.get(student_num=student_num).delete()
for plist in root[1]:
try:
if PlistCutoff.objects.filter(year=int(plist[0].text)).exists():
p = PlistCutoff.objects.get(year=int(plist[0].text))
p.grade_8_T1 = float(plist[1].text)
p.grade_8_T2 = float(plist[2].text)
p.grade_9_T1 = float(plist[3].text)
p.grade_9_T2 = float(plist[4].text)
p.grade_10_T1 = float(plist[5].text)
p.grade_10_T2 = float(plist[6].text)
p.grade_11_T1 = float(plist[7].text)
p.grade_11_T2 = float(plist[8].text)
p.grade_12_T1 = float(plist[9].text)
p.grade_12_T2 = float(plist[10].text)
p.save(user)
else:
PlistCutoff.objects.get(year=int(plist[0].text),
grade_8_T1=float(plist[1].text), grade_8_T2=float(plist[2].text),
grade_9_T1=float(plist[3].text), grade_9_T2=float(plist[4].text),
grade_10_T1=float(plist[5].text), grade_10_T2=float(plist[6].text),
grade_11_T1=float(plist[7].text), grade_11_T2=float(plist[8].text),
grade_12_T1=float(plist[9].text), grade_12_T2=float(plist[10].text),
).save(user)
logs.append(f"Configured Plist Cutoffs for {plist[0].text}-{int(plist[0].text) + 1}")
except Exception as e:
logs.append(f"Failed to add Plist Cutoffs for {plist[0].text}-{int(plist[0].text) + 1}: {e}")
for c in root[2]:
try:
if PointCodes.objects.filter(catagory=c[2].text, code=int(c[0].text)).exists():
obj = PointCodes.objects.get(catagory=c[2].text, code=int(c[0].text))
obj.description = c[1].text
obj.save()
else:
PointCodes.objects.create(catagory=c[2].text, code=int(c[0].text), description=c[1].text)
logs.append(
f"Created code definition for {c[2].text}{c[0].text} with desc. {c[1].text}")
except Exception as e:
logs.append(f"Failed to create code definition for {c[2].text}{c[0].text} with desc. {c[1].text}: {e}")
done = True
close_old_connections()
def IdleUnmountProc(self, q):
while 1:
try:
try:
if q.get_nowait() == 'STOP':
break
except Empty:
pass
numreadydrives = robotdrives.objects.filter(
status='Ready').count()
robotdrives_objs = robotdrives.objects.filter(status='Mounted')
if robotdrives_objs:
#################################################
# Found mounted tape
nummountreq = robotQueue.objects.filter(ReqType=50,
Status=0).count()
for robotdrives_obj in robotdrives_objs:
if len(robotdrives_obj.drive_lock) == 0:
#################################################
# Tape is not locked
if int(
robotdrives_obj.IdleTime
) > 10 and nummountreq > 0 and numreadydrives == 0:
##################################################################################################
# Pendning mount request in RobotReqQueue. Set IdleTime to 10 sec.
robotdrives_obj.IdleTime = 10
robotdrives_obj.save(
update_fields=['IdleTime'])
logger.info('Setting IdleTime to ' +
str(robotdrives_obj.IdleTime) +
' sec, for tape id: ' +
str(robotdrives_obj.t_id))
elif int(robotdrives_obj.IdleTime) == 9999:
##################################################################
# Tape is new_unlocked(9999). Set IdleTime to 3600 sec.
robotdrives_obj.IdleTime = 3600
robotdrives_obj.save(
update_fields=['IdleTime'])
logger.info('Setting IdleTime to ' +
str(robotdrives_obj.IdleTime) +
' sec, for tape id: ' +
str(robotdrives_obj.t_id))
if int(robotdrives_obj.IdleTime) < 9999:
########################################################################
# Tape is already discoverd as unlocked(<9999). Count down IdleTime with 1 sec.
robotdrives_obj.IdleTime -= 1
robotdrives_obj.save(
update_fields=['IdleTime'])
logger.debug(
'Count down -1 sec setting IdleTime to ' +
str(robotdrives_obj.IdleTime) +
' sec, for tape id: ' +
str(robotdrives_obj.t_id))
if int(robotdrives_obj.IdleTime) <= 0:
####################################################
# Tape IdleTime is zero(0). Try to unmount the tape
logger.info('Start to unmount tape: ' +
str(robotdrives_obj.t_id))
robotQueue_obj = robotQueue()
robotQueue_obj.ReqType = 51 # Unmount
robotQueue_obj.ReqPurpose = 'TLD - IdleUnmountProc'
robotQueue_obj.Status = 0 # Pending
robotQueue_obj.MediumID = robotdrives_obj.t_id
robotQueue_obj.user = '******'
robotQueue_obj.save()
while 1:
robot_objs = robot.objects.filter(
t_id=robotdrives_obj.t_id,
drive_id='99')
if robot_objs:
logger.info(
'Success to unmount tape: ' +
str(robotdrives_obj.t_id))
break
else:
if not robotQueue.objects.filter(
id=robotQueue_obj.id).exists():
logger.warning(
'Add a second unmount request for: %s'
% robotdrives_obj.t_id)
robotQueue_obj = robotQueue()
robotQueue_obj.ReqType = 51 # Unmount
robotQueue_obj.ReqPurpose = 'TLD - IdleUnmountProc'
robotQueue_obj.Status = 0 # Pending
robotQueue_obj.MediumID = robotdrives_obj.t_id
robotQueue_obj.user = '******'
robotQueue_obj.save()
time.sleep(1)
else:
logger.debug(
'Wait for unmounting of: ' +
str(robotdrives_obj.t_id))
time.sleep(10)
else:
#################################################
# Tape is locked. Set IdleTime to 9999.
robotdrives_obj.IdleTime = 9999
robotdrives_obj.save(update_fields=['IdleTime'])
logger.debug(
'Tape is locked setting IdleTime to ' +
str(robotdrives_obj.IdleTime) +
' sec, for tape id: ' +
str(robotdrives_obj.t_id))
db.close_old_connections()
time.sleep(1)
except:
logger.error('Unexpected error: %s' % (str(sys.exc_info())))
raise
logger.info('Idle Unmount Process stopped')
def close_connections():
close_old_connections()
def update_backend(self, label, using):
backend = haystack_connections[using].get_backend()
unified_index = haystack_connections[using].get_unified_index()
for model in haystack_get_models(label):
try:
index = unified_index.get_index(model)
except NotHandled:
if self.verbosity >= 2:
self.stdout.write("Skipping '%s' - no index." % model)
continue
if self.workers > 0:
# workers resetting connections leads to references to models / connections getting
# stale and having their connection disconnected from under them. Resetting before
# the loop continues and it accesses the ORM makes it better.
close_old_connections()
qs = index.build_queryset(using=using,
start_date=self.start_date,
end_date=self.end_date)
total = qs.count()
if self.verbosity >= 1:
self.stdout.write(
u"Indexing %d %s" %
(total, force_text(model._meta.verbose_name_plural)))
batch_size = self.batchsize or backend.batch_size
if self.workers > 0:
ghetto_queue = []
max_pk = None
for start in range(0, total, batch_size):
end = min(start + batch_size, total)
if self.workers == 0:
max_pk = do_update(backend,
index,
qs,
start,
end,
total,
verbosity=self.verbosity,
commit=self.commit,
max_retries=self.max_retries,
last_max_pk=max_pk)
else:
ghetto_queue.append(
(model, start, end, total, using, self.start_date,
self.end_date, self.verbosity, self.commit,
self.max_retries))
if self.workers > 0:
pool = multiprocessing.Pool(self.workers)
successful_tasks = pool.map(update_worker, ghetto_queue)
if len(ghetto_queue) != len(successful_tasks):
self.stderr.write(
'Queued %d tasks but only %d completed' %
(len(ghetto_queue), len(successful_tasks)))
for i in ghetto_queue:
if i not in successful_tasks:
self.stderr.write('Incomplete task: %s' % repr(i))
pool.close()
pool.join()
if self.remove:
if self.start_date or self.end_date or total <= 0:
# They're using a reduced set, which may not incorporate
# all pks. Rebuild the list with everything.
qs = index.index_queryset().values_list('pk', flat=True)
database_pks = set(smart_bytes(pk) for pk in qs)
total = len(database_pks)
else:
database_pks = set(
smart_bytes(pk)
for pk in qs.values_list('pk', flat=True))
# Since records may still be in the search index but not the local database
# we'll use that to create batches for processing.
# See https://github.com/django-haystack/django-haystack/issues/1186
index_total = SearchQuerySet(
using=backend.connection_alias).models(model).count()
# Retrieve PKs from the index. Note that this cannot be a numeric range query because although
# pks are normally numeric they can be non-numeric UUIDs or other custom values. To reduce
# load on the search engine, we only retrieve the pk field, which will be checked against the
# full list obtained from the database, and the id field, which will be used to delete the
# record should it be found to be stale.
index_pks = SearchQuerySet(
using=backend.connection_alias).models(model)
index_pks = index_pks.values_list('pk', 'id')
# We'll collect all of the record IDs which are no longer present in the database and delete
# them after walking the entire index. This uses more memory than the incremental approach but
# avoids needing the pagination logic below to account for both commit modes:
stale_records = set()
for start in range(0, index_total, batch_size):
upper_bound = start + batch_size
# If the database pk is no longer present, queue the index key for removal:
for pk, rec_id in index_pks[start:upper_bound]:
if smart_bytes(pk) not in database_pks:
stale_records.add(rec_id)
if stale_records:
if self.verbosity >= 1:
self.stdout.write(" removing %d stale records." %
len(stale_records))
for rec_id in stale_records:
# Since the PK was not in the database list, we'll delete the record from the search
# index:
if self.verbosity >= 2:
self.stdout.write(" removing %s." % rec_id)
backend.remove(rec_id, commit=self.commit)
def archive(archive_id):
"""
执行数据库归档
:return:
"""
archive_info = ArchiveConfig.objects.get(id=archive_id)
s_ins = archive_info.src_instance
src_db_name = archive_info.src_db_name
src_table_name = archive_info.src_table_name
condition = archive_info.condition
no_delete = archive_info.no_delete
sleep = archive_info.sleep
mode = archive_info.mode
# 获取归档表的字符集信息
s_engine = get_engine(s_ins)
s_db = s_engine.schema_object.databases[src_db_name]
s_tb = s_db.tables[src_table_name]
s_charset = s_tb.options['charset'].value
if s_charset is None:
s_charset = s_db.options['charset'].value
pt_archiver = PtArchiver()
# 准备参数
source = fr"h={s_ins.host},u={s_ins.user},p={s_ins.password}," \
fr"P={s_ins.port},D={src_db_name},t={src_table_name},A={s_charset}"
args = {
"no-version-check": True,
"source": source,
"where": condition,
"progress": 5000,
"statistics": True,
"charset": 'utf8',
"limit": 10000,
"txn-size": 1000,
"sleep": sleep
}
# 归档到目标实例
if mode == 'dest':
d_ins = archive_info.dest_instance
dest_db_name = archive_info.dest_db_name
dest_table_name = archive_info.dest_table_name
# 目标表的字符集信息
d_engine = get_engine(d_ins)
d_db = d_engine.schema_object.databases[dest_db_name]
d_tb = d_db.tables[dest_table_name]
d_charset = d_tb.options['charset'].value
if d_charset is None:
d_charset = d_db.options['charset'].value
# dest
dest = fr"h={d_ins.host},u={d_ins.user},p={d_ins.password},P={d_ins.port}," \
fr"D={dest_db_name},t={dest_table_name},A={d_charset}"
args['dest'] = dest
if no_delete:
args['no-delete'] = True
elif mode == 'file':
output_directory = os.path.join(settings.BASE_DIR,
'downloads/archiver')
os.makedirs(output_directory, exist_ok=True)
args[
'file'] = f'{output_directory}/{s_ins.instance_name}-{src_db_name}-{src_table_name}.txt'
if no_delete:
args['no-delete'] = True
elif mode == 'purge':
args['purge'] = True
# 参数检查
args_check_result = pt_archiver.check_args(args)
if args_check_result['status'] == 1:
return JsonResponse(args_check_result)
# 参数转换
cmd_args = pt_archiver.generate_args2cmd(args, shell=True)
# 执行命令，获取结果
select_cnt = 0
insert_cnt = 0
delete_cnt = 0
with FuncTimer() as t:
p = pt_archiver.execute_cmd(cmd_args, shell=True)
stdout = ''
for line in iter(p.stdout.readline, ''):
if re.match(r'^SELECT\s(\d+)$', line, re.I):
select_cnt = re.findall(r'^SELECT\s(\d+)$', line)
elif re.match(r'^INSERT\s(\d+)$', line, re.I):
insert_cnt = re.findall(r'^INSERT\s(\d+)$', line)
elif re.match(r'^DELETE\s(\d+)$', line, re.I):
delete_cnt = re.findall(r'^DELETE\s(\d+)$', line)
stdout += f'{line}\n'
statistics = stdout
# 获取异常信息
stderr = p.stderr.read()
if stderr:
statistics = stdout + stderr
# 判断归档结果
select_cnt = int(select_cnt[0]) if select_cnt else 0
insert_cnt = int(insert_cnt[0]) if insert_cnt else 0
delete_cnt = int(delete_cnt[0]) if delete_cnt else 0
error_info = ''
success = True
if stderr:
error_info = f'命令执行报错:{stderr}'
success = False
if mode == 'dest':
# 删除源数据，判断删除数量和写入数量
if not no_delete and (insert_cnt != delete_cnt):
error_info = f"删除和写入数量不一致:{insert_cnt}!={delete_cnt}"
success = False
elif mode == 'file':
# 删除源数据，判断查询数量和删除数量
if not no_delete and (select_cnt != delete_cnt):
error_info = f"查询和删除数量不一致:{select_cnt}!={delete_cnt}"
success = False
elif mode == 'purge':
# 直接删除。判断查询数量和删除数量
if select_cnt != delete_cnt:
error_info = f"查询和删除数量不一致:{select_cnt}!={delete_cnt}"
success = False
# 执行信息保存到数据库
if connection.connection and not connection.is_usable():
close_old_connections()
# 更新最后归档时间
ArchiveConfig(
id=archive_id,
last_archive_time=t.end).save(update_fields=['last_archive_time'])
# 替换密码信息后保存
ArchiveLog.objects.create(
archive=archive_info,
cmd=cmd_args.replace(s_ins.password, '***').replace(
d_ins.password, '***') if mode == 'dest' else cmd_args.replace(
s_ins.password, '***'),
condition=condition,
mode=mode,
no_delete=no_delete,
sleep=sleep,
select_cnt=select_cnt,
insert_cnt=insert_cnt,
delete_cnt=delete_cnt,
statistics=statistics,
success=success,
error_info=error_info,
start_time=t.start,
end_time=t.end)
if not success:
raise Exception(f'{error_info}\n{statistics}')
def run(self):
with transaction.atomic():
Task.objects.next()
self.locked.set()
self.exit.wait(timeout=5)
close_old_connections()
def goods_out(uc_shopid, template_shop_ids, batch_id, days):
"""
输出选品结果给运营angela查看
:return:
"""
# conn_ucenter = connections['ucenter']
# cursor_ucenter = conn_ucenter.cursor()
conn_ai = connections['default']
cursor_ai = conn_ai.cursor()
# conn_dmstore = connections['dmstore']
# cursor_dmstore = conn_dmstore.cursor()
print(
"时间,门店id,门店名称,一级分类,二级分类,三级分类,四级分类,配送类型,商品编码,商品名称,商品upc,策略标签,商品角色,上品优先级排名,商品实际销售4周预期psd金额,商品实际销售4周预期psd,组内门店4周预期psd金额,组内门店4周预期psd,关联分值"
)
select_sql = "select * from goods_goodsselectionhistory where uc_shopid={} and batch_id='{}' and upc is not NULL and goods_role in (1,2,4,0,3)"
select_sql_02 = "select mch_goods_code from goods_goodsselectionhistory where uc_shopid={} and batch_id='{}' and upc is NULL and goods_role=3"
select_sql_03 = "SELECT mch_goods_code,relation_score from goods_goodsselectionhistory WHERE uc_shopid={} and batch_id='{}'"
cursor_ai.execute(select_sql.format(uc_shopid, batch_id))
all_data = cursor_ai.fetchall()
# tt = []
# for i in all_data:
# tt.append(i[10])
cursor_ai.execute(select_sql_02.format(uc_shopid, batch_id))
all_data_02 = cursor_ai.fetchall()
cursor_ai.execute(select_sql_03.format(uc_shopid, batch_id))
relation_score = cursor_ai.fetchall()
relation_score_dict = {}
for d in relation_score:
if d[1]:
relation_score_dict[d[0]] = d[1]
conn_ai.close()
tem = ""
for data in all_data[:]:
close_old_connections()
conn_ucenter = connections['ucenter']
cursor_ucenter = conn_ucenter.cursor()
conn_dmstore = connections['dmstore']
cursor_dmstore = conn_dmstore.cursor()
# print('==================================================')
# if not data[19] in [ 1, 3]:
# continue
# print('data',data)
"时间,门店id,门店名称,一级分类,二级分类,三级分类,配送类型,商品编码,商品名称,商品upc,策略标签,商品角色 ,上品优先级排名,商品实际销售4周预期psd,商品实际销售4周预期psd金额,组内门店4周预期psd 组内门店4周预期psd金额 全店4周预期psd 全店4周预期psd金额"
line_str = "" # 一条记录
line_str += str(data[12]) # 时间
line_str += ","
line_str += str(data[2]) #门店id
line_str += ","
shop_name_sql = "select shop_name from uc_shop a where id={}"
# close_old_connections()
cursor_ucenter.execute(shop_name_sql.format(uc_shopid))
try:
line_str += str(cursor_ucenter.fetchone()[0]) #门店名称
except:
line_str += str('None') # 门店名称
tem = line_str
line_str += ","
class_type_sql = "select display_first_cat_id,display_second_cat_id,display_third_cat_id,display_fourth_cat_id,delivery_type from uc_merchant_goods where mch_goods_code='{}' and width > 0"
cursor_ucenter.execute(class_type_sql.format(str(data[10])))
class_type_data_all = cursor_ucenter.fetchall()
try:
class_type_data = class_type_data_all[0]
# print(i)
if len(class_type_data_all) > 1:
for d in class_type_data_all:
if d[0] != '0' and d[0] != '':
class_type_data = d
except:
class_type_data = None
# class_type_sql = "SELECT DISTINCT first_cate_id,second_cate_id,third_cate_id from goods WHERE neighbor_goods_id ={} AND corp_id=2"
# cursor_dmstore.execute(class_type_sql.format(data[10]))
# class_type_data = cursor_dmstore.fetchone()
# print('分类',data[10],class_type_data)
try:
line_str += str(class_type_data[0]) # 一级分类
line_str += ","
except:
line_str += str('None')
line_str += ","
try:
line_str += str(class_type_data[1]) # 二级分类
line_str += ","
except:
line_str += str('None')
line_str += ","
try:
line_str += str(class_type_data[2]) # 三级分类
line_str += ","
except:
line_str += str('None')
line_str += ","
try:
line_str += str(class_type_data[3]) # 四级分类
line_str += ","
except:
line_str += str('None')
line_str += ","
delivery_type_sql = "select DISTINCT a.supplier_goods_code,b.delivery_attr from uc_supplier_goods a LEFT JOIN uc_supplier_delivery b on a.delivery_type=b.delivery_code where a.supplier_id = 1 and order_status = 1 AND supplier_goods_code={}"
cursor_ucenter.execute(delivery_type_sql.format(data[10]))
delivery_type_dict = {1: '日配', 2: '非日配', '1': '日配', '2': '非日配'}
try:
delivery = cursor_ucenter.fetchone()[1]
# print(data[21])
# line_str += str(delivery_type_dict[data[21]]) # 配送类型
line_str += str(delivery_type_dict[delivery]) # 配送类型
line_str += ","
except:
line_str += str('None')
line_str += ","
line_str += str(data[10]) # 商品编码
line_str += ","
line_str += str(data[5]) # 商品名称
line_str += ","
line_str += str(data[4]) # 商品upc
line_str += ","
#策略标签
which_strategy_dict = {
0: '结构品',
1: '畅销品',
2: '关联品',
3: '品库可定商品',
4: '品谱选品',
5: '决策树标签选品',
6: '人工临时加品',
7: '网红品',
101: '保留策略',
201: '原陈列非保留品',
301: '品库可订商品',
401: '不可订货商品'
}
# print('data[19]',data[19])
# if data[19] == 1:
try:
if data[15] == 1:
line_str += str(which_strategy_dict[0]) # 策略标签
elif data[16] == 1:
line_str += str(which_strategy_dict[1]) # 策略标签
elif data[17] == 1:
line_str += str(which_strategy_dict[2]) # 策略标签
elif data[19] in [0, 2, 3, 4]:
line_str += str(which_strategy_dict[data[20]]) # 策略标签
else:
line_str += str('None') # 策略标签
except:
line_str += str('None') # 策略标签
# else:
# line_str += str('None') # 策略标签
line_str += ","
# if data[19] == 1:
# try:
# line_str += str(which_strategy_dict[data[20]]) # 策略标签
# except:
# line_str += str('None') # 策略标签
# else:
# line_str += str('None') # 策略标签
# line_str += ","
# 商品角色
goods_role_dict = {0: '保护品', 1: '必上', 2: '必下', 3: '可选上架', 4: '可选下架'}
if data[19] in [0, 1, 2, 3, 4]:
try:
line_str += str(goods_role_dict[data[19]]) # 商品角色
except:
line_str += str('None') # 商品角色
else:
line_str += str('None') # 商品角色
line_str += ","
# 上品优先级排名
if data[19] == 1 or data[19] == 3:
try:
line_str += str(data[14]) # 上品优先级排名
except:
line_str += str('None') #上品优先级排名
else:
line_str += str('None') # 上品优先级排名
line_str += ","
# now = datetime.datetime.now()
# now_date = now.strftime('%Y-%m-%d %H:%M:%S')
now_date = data[12].strftime('%Y-%m-%d %H:%M:%S')
week_ago = (
data[12] -
datetime.timedelta(days=days)).strftime('%Y-%m-%d %H:%M:%S')
#商品实际销售4周预期psd,商品实际销售4周预期psd金额,组内门店4周预期psd 组内门店4周预期psd金额 全店4周预期psd 全店4周预期psd金额
psd_sql = "select sum(p.amount),g.first_cate_id,g.second_cate_id,g.third_cate_id,g.price,p.name from dmstore.payment_detail as p left join dmstore.goods as g on p.goods_id=g.id where p.create_time > '{}' and p.create_time < '{}' and p.shop_id ={} and g.neighbor_goods_id={};"
# close_old_connections()
cursor_dmstore.execute(
psd_sql.format(week_ago, now_date, data[1], data[10]))
# print(psd_sql.format(week_ago,now_date,data[1],data[10]))
psd_data = cursor_dmstore.fetchone()
# print('psd_data',psd_data)
if psd_data[0]:
line_str += str(psd_data[0] / days) # psd金额
line_str += ","
try:
line_str += str(psd_data[0] / (days * psd_data[4])) # psd
except:
line_str += str(0) # psd
line_str += ","
else:
line_str += str(0) # psd金额
line_str += ","
line_str += str(0) # psd
line_str += ","
psd_sql_shops = "select sum(p.amount), COUNT(DISTINCT shop_id),g.price,p.name from dmstore.payment_detail as p left join dmstore.goods as g on p.goods_id=g.id where p.create_time > '{}' and p.create_time < '{}' and p.shop_id in ({}) and g.neighbor_goods_id={};"
# close_old_connections()
cursor_dmstore.execute(
psd_sql_shops.format(week_ago, now_date, template_shop_ids,
data[10]))
psd_data_shops = cursor_dmstore.fetchone()
# print('psd_data_shops',psd_data_shops)
if psd_data_shops[0]:
line_str += str(psd_data_shops[0] /
(days * psd_data_shops[1])) # psd金额,同组
line_str += ","
try:
line_str += str(
psd_data_shops[0] /
(days * psd_data_shops[1] * psd_data_shops[2])) # psd,同组
except:
line_str += str(0)
line_str += ","
else:
line_str += str(0) # psd金额,同组
line_str += ","
line_str += str(0) # psd,同组
line_str += ","
try:
line_str += str(data[22])
except:
line_str += str(0)
print(line_str)
conn_ucenter.close()
conn_dmstore.close()
tem_mch_list = [i[0] for i in all_data_02]
conn_ucenter = connections['ucenter']
cursor_ucenter = conn_ucenter.cursor()
sql2 = "SELECT mch_goods_code,upc ,goods_name,display_first_cat_id,display_second_cat_id,display_third_cat_id,display_fourth_cat_id,delivery_type from uc_merchant_goods where mch_goods_code ='{}'"
# d = cursor_ucenter.fetchall()
# print("订货0的mch的len", len(tem_mch_list))
# print("订货0的len", len(d))
for t in tem_mch_list[:]:
try:
cursor_ucenter.execute(sql2.format(str(t)))
data = cursor_ucenter.fetchall()
i = data[0]
# print(i)
if len(data) > 1:
for d in data:
if d[3] != '0' and d[3] != '':
i = d
delivery_type_sql = "select DISTINCT a.supplier_goods_code,b.delivery_attr from uc_supplier_goods a LEFT JOIN uc_supplier_delivery b on a.delivery_type=b.delivery_code where a.supplier_id = 1 and order_status = 1 AND supplier_goods_code={}"
cursor_ucenter.execute(delivery_type_sql.format(t))
delivery_type_dict = {1: '日配', 2: '非日配', '1': '日配', '2': '非日配'}
delivery_str = ''
try:
delivery = cursor_ucenter.fetchone()[1]
delivery_str = str(delivery_type_dict[delivery]) # 配送类型
except:
delivery_str = str('None')
try:
relation_score_value = relation_score_dict[i[0]]
except:
relation_score_value = 0
print("{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}".format(
tem, i[3], i[4], i[5], i[6], delivery_str, i[0], i[2], i[1],
'品库可订商品', '可选上架', 0, None, None, None, None,
relation_score_value))
except:
continue
conn_ucenter.close()
def handle(self, *args, **options):
start = datetime.now() # Measure the time it takes to run the script.
day = options['date']
if not day:
day = (datetime.now() - timedelta(days=1)).strftime('%Y-%m-%d')
sep = options['separator']
if options['stats_source'] == 's3':
filepath = 's3://' + '/'.join([
settings.AWS_STATS_S3_BUCKET, settings.AWS_STATS_S3_PREFIX,
'download_counts', day, '000000_0'
])
elif options['stats_source'] == 'file':
folder = options['folder_name']
folder = path.join(settings.TMP_PATH, folder, day)
filepath = path.join(folder, 'download_counts.hive')
# Make sure we're not trying to update with mismatched data.
if get_date(filepath, sep) != day:
raise CommandError('%s file contains data for another day' %
filepath)
# First, make sure we don't have any existing counts for the same day,
# or it would just increment again the same data.
DownloadCount.objects.filter(date=day).delete()
# Memoize the files to addon relations and the DownloadCounts.
download_counts = {}
# Perf: preload all the files and slugs once and for all.
# This builds two dicts:
# - One where each key (the file_id we get from the hive query) has
# the addon_id as value.
# - One where each key (the add-on slug) has the add-on_id as value.
files_to_addon = dict(
File.objects.values_list('id', 'version__addon_id'))
slugs_to_addon = dict(
Addon.unfiltered.exclude(status=amo.STATUS_NULL).values_list(
'slug', 'id'))
# Only accept valid sources, which are constants. The source must
# either be exactly one of the "full" valid sources, or prefixed by one
# of the "prefix" valid sources.
fulls = amo.DOWNLOAD_SOURCES_FULL
prefixes = amo.DOWNLOAD_SOURCES_PREFIX
count_file = get_stats_data(filepath)
for index, line in enumerate(count_file):
if index and (index % 1000000) == 0:
log.info('Processed %s lines' % index)
splitted = line[:-1].split(sep)
if len(splitted) != 4:
log.debug('Badly formatted row: %s' % line)
continue
day, counter, id_or_slug, src = splitted
try:
# Clean up data.
id_or_slug = id_or_slug.strip()
counter = int(counter)
except ValueError:
# Ignore completely invalid data.
continue
if id_or_slug.strip().isdigit():
# If it's a digit, then it should be a file id.
try:
id_or_slug = int(id_or_slug)
except ValueError:
continue
# Does this file exist?
if id_or_slug in files_to_addon:
addon_id = files_to_addon[id_or_slug]
# Maybe it's an add-on ?
elif id_or_slug in files_to_addon.values():
addon_id = id_or_slug
else:
# It's an integer we don't recognize, ignore the row.
continue
else:
# It's probably a slug.
if id_or_slug in slugs_to_addon:
addon_id = slugs_to_addon[id_or_slug]
else:
# We've exhausted all possibilities, ignore this row.
continue
if not is_valid_source(src, fulls=fulls, prefixes=prefixes):
continue
# Memoize the DownloadCount.
if addon_id in download_counts:
dc = download_counts[addon_id]
else:
dc = DownloadCount(date=day, addon_id=addon_id, count=0)
download_counts[addon_id] = dc
# We can now fill the DownloadCount object.
dc.count += counter
dc.sources = update_inc(dc.sources, src, counter)
# Close all old connections in this thread before we start creating the
# `DownloadCount` values.
# https://github.com/mozilla/addons-server/issues/6886
# If the calculation above takes too long it might happen that we run
# into `wait_timeout` problems and django doesn't reconnect properly
# (potentially because of misconfiguration).
# Django will re-connect properly after it notices that all
# connections are closed.
close_old_connections()
# Create in bulk: this is much faster.
DownloadCount.objects.bulk_create(download_counts.values(), 100)
log.info('Processed a total of %s lines' % (index + 1))
log.debug('Total processing time: %s' % (datetime.now() - start))
if options['stats_source'] == 'file':
# Clean up file.
log.debug('Deleting {path}'.format(path=filepath))
unlink(filepath)
def _cb(self, status):
# when using mysql, we need to reconnect because this is running in a new process
db.close_old_connections()
t = Task.objects.get(pk=self._task_id)
t.status = status
t.save()
