def startMethodAndJoin(servmethod, param, url=None): thread = SyncThread() thread.callBackURL = url thread.startMethod(servmethod,param) thread.join() connection.close() return
def run_select_for_update(self, status, nowait=False): """ Utility method that runs a SELECT FOR UPDATE against all Person instances. After the select_for_update, it attempts to update the name of the only record, save, and commit. This function expects to run in a separate thread. """ status.append('started') try: # We need to enter transaction management again, as this is done on # per-thread basis transaction.enter_transaction_management() people = list( Person.objects.all().select_for_update(nowait=nowait) ) people[0].name = 'Fred' people[0].save() transaction.commit() except DatabaseError as e: status.append(e) finally: # This method is run in a separate thread. It uses its own # database connection. Close it without waiting for the GC. transaction.abort() connection.close()
def handle(self, path_template=None, **options): v = int(options.get('verbosity', 0)) if v == 0: logger.level = logging.ERROR elif v == 1: logger.level = logging.INFO elif v >= 2: logger.level = logging.DEBUG if not path_template: path_template = '/srv/ftp/%(repo)s/os/%(arch)s/' self.path_template = path_template notifier = self.setup_notifier() # this thread is done using the database; all future access is done in # the spawned read_repo() processes, so close the otherwise completely # idle connection. connection.close() logger.info('Entering notifier loop') notifier.loop() logger.info('Cancelling remaining threads...') for thread in threading.enumerate(): if hasattr(thread, 'cancel'): thread.cancel()
def added_mem(request, **kwargs): """ 查看自己添加的会员 """ template_name = kwargs.pop("template_name") right = "store/adder.html" UserBasic = kwargs.pop("Mem") UserAdder = user_adder.objects.filter(adding = UserBasic) from django.db import connection cursor = connection.cursor() cursor.execute("SELECT BAS.number , BAS.name , LM.name ,MID.init_money , BAS.start_date FROM `rtyk_triple`.`member_user_adder` AD INNER JOIN `rtyk_triple`.`member_user_mid_mem` MID ON MID.user_id = AD.added_id INNER JOIN `rtyk_triple`.`member_user_basic` BAS ON BAS.id = AD.added_id INNER JOIN `rtyk_triple`.`management_member_lv_money` LM ON LM.id = MID.level_id WHERE unix_timestamp(start_date) BETWEEN unix_timestamp(CURRENT_DATE - INTERVAL 3 MONTH) AND unix_timestamp(CURRENT_DATE) and AD.adding_id = %s",UserBasic.id) adding_mem = cursor.fetchall() connection.close() cursor.close() adding_mem = PaginatorFuc(request,adding_mem,number=20) # print adding_mem # # if UserAdder: # for UAdder in UserAdder: # UserMidMem = user_mid_mem.objects.filter(user = UAdder.added)[0] # UAdder.level_name = UserMidMem.level.name # UAdder.money = UserMidMem.level.money ctx = { 'UserBasic': UserBasic, 'user_adder': adding_mem, 'right': right, 'user_number':UserBasic.number } return render_to_response(template_name,RequestContext(request,ctx))
def handle_one(check): """ Send an alert for a single check. Return True if an appropriate check was selected and processed. Return False if no checks need to be processed. """ check.status = check.get_status() tmpl = "Sending alert, status=%s, code=%s\n" _stdout(tmpl % (check.status, check.code)) # Save the new status. check.status = check.get_status() check.save() tmpl = "\nSending alert, status=%s, code=%s\n" _stdout(tmpl % (check.status, check.code)) try: errors = check.send_alert() except: agent.record_exception() finally: connection.close() for ch, error in errors: _stdout("ERROR: %s %s %s\n" % (ch.kind, ch.value, error)) return True
def version_create(request, package_name): """ The form will create versions according to permissions, plugin name and description are updated according to the info contained in the package metadata """ plugin = get_object_or_404(Plugin, package_name=package_name) if not check_plugin_access(request.user, plugin): return render_to_response('plugins/version_permission_deny.html', { 'plugin' : plugin }, context_instance=RequestContext(request)) version = PluginVersion(plugin = plugin, created_by = request.user) if request.method == 'POST': form = PluginVersionForm(request.POST, request.FILES, instance=version, is_trusted=request.user.has_perm('plugins.can_approve')) if form.is_valid(): try: new_object = form.save() msg = _("The Plugin Version has been successfully created.") messages.success(request, msg, fail_silently=True) # The approved flag is also controlled in the form, but we # are checking it here in any case for additional security if not request.user.has_perm('plugins.can_approve'): new_object.approved = False new_object.save() messages.warning(request, _('You do not have approval permissions, plugin version has been set unapproved.'), fail_silently=True) version_notify(new_object) if form.cleaned_data.get('icon_file'): form.cleaned_data['icon'] = form.cleaned_data.get('icon_file') _main_plugin_update(request, new_object.plugin, form) _check_optional_metadata(form, request) return HttpResponseRedirect(new_object.plugin.get_absolute_url()) except (IntegrityError, ValidationError, DjangoUnicodeDecodeError), e: messages.error(request, e, fail_silently=True) connection.close() return HttpResponseRedirect(plugin.get_absolute_url())
def handle(self, **options): if options['index_type'] == 'feature': feature_index = site.get_index(Feature) features = Feature.objects.filter(published=True) back.update(feature_index, features) else: recipient_index = site.get_index(Recipient) location_index = site.get_index(Location) if options['country']: index_data = Recipient.objects.select_related().filter(countrypayment=options['country'], total__gt=1000).only('name', 'geo1', 'geo2', 'geo3', 'geo4', 'zipcode', 'countrypayment') locations = Location.objects.filter(country=options['country']) else: raise ValueError('Country is required') settings.HAYSTACK_XAPIAN_PATH = "%s-%s" % (settings.HAYSTACK_XAPIAN_PATH, options['country']) back = backend.SearchBackend() print "now indexing Recipients" back.update(recipient_index, index_data) print "now indexing Location" back.update(location_index, locations) connection.close()
def set_many(self): usage_key = self._gen_usage_key() self.client.get_many(self.client.username, [usage_key]) self.client.set_many(self.client.username, {usage_key: self._gen_block_data()}) self.usages_with_data.add(usage_key) transaction.commit() connection.close()
def test_close(self): with self.store_signals() as signals: connection.close() self.run_query() self.assertEqual(len(signals), 2) self.assertEqual('pre_close', signals[0][0]) self.assertEqual('post_close', signals[1][0])
def refresh(stream=sys.stdout, dumpfile=None, create=False, quiet=False, yes=None, pgformat=False): """Apply a dump file, dropping and creating database.""" if not dumpfile or not os.path.exists(dumpfile): raise CuckooError('You must provide a valid dump file: %s' % dumpfile) connection.close() cuckoo_db_name = getattr(settings, 'CUCKOO_DB', 'default') env = settings.DATABASES[cuckoo_db_name] dropcmd = get_db_shell_cmd(cuckoo_db_name, True, True, 'dropdb') if database_exists(env['NAME']): print '[CUCKOO] Dropping database.' checked_call(dropcmd, "could not drop database") if create: print '[CUCKOO] Creating database.' createcmd = get_db_shell_cmd(cuckoo_db_name, False, True, 'createdb') checked_call(createcmd, "Could not create database") print '[CUCKOO] Applying dump file: %s' % dumpfile try: if pgformat: restorecmd = get_db_shell_cmd(cuckoo_db_name, True, True, 'pg_restore -e -Fc -j 4') restorecmd += ' %s' % dumpfile print restorecmd checked_call(restorecmd, "could not restore database") else: output = _execute_file(cuckoo_db_name, dumpfile, exists=create, dba=True) if not quiet: print output except Exception as e: raise CuckooError("[CUCKOO] Error while executing dump file %s\n %s" % (dumpfile, e))
def handle(self, **options): '''The main entry point for the Django management command.''' import_start = datetime.datetime.now() # Start transaction management. transaction.commit_unless_managed() transaction.enter_transaction_management() transaction.managed(True) try: self._promote_devel() except: self._rollback_db() raise # Finalize the transaction and close the db connection. transaction.commit() transaction.leave_transaction_management() connection.close() import_end = datetime.datetime.now() # Print a short summary of what we did. td = import_end - import_start print '\nProcessing complete in %s days, %s.%s seconds.' % ( td.days, td.seconds, td.microseconds) print ' TraitData objects promoted: %s' % ( PublicTraitData.objects.all().count(),)
def error_rate(instrument_id, n_hours=24): """ Returns the rate of errors for the last n_hours hours. @param instrument_id: Instrument model object @param n_hours: number of hours to track """ # Try calling the stored procedure (faster) try: cursor = connection.cursor() cursor.callproc("error_rate", (instrument_id.id,)) msg = cursor.fetchone()[0] cursor.execute('FETCH ALL IN "%s"' % msg) rows = cursor.fetchall() cursor.close() return [[int(r[0]), int(r[1])] for r in rows] except: connection.close() logging.error("Error rate (%s): %s", str(instrument_id), sys.exc_value) # Do it by hand (slow) time = timezone.now() errors = [] running_sum = 0 for i in range(n_hours): t_i = time - datetime.timedelta(hours=i + 1) n = Error.objects.filter(run_status_id__run_id__instrument_id=instrument_id, run_status_id__created_on__gte=t_i).count() n -= running_sum running_sum += n errors.append([-i, n]) return errors
def create_test_spatial_db(verbosity=1, autoclobber=False, interactive=False): "Creates a spatial database based on the settings." # Making sure we're using PostgreSQL and psycopg2 if settings.DATABASE_ENGINE != 'sqlite3': raise Exception('SpatiaLite database creation only supported on sqlite3 platform.') # Getting the test database name using the the SQLite backend's # `_create_test_db`. Unless `TEST_DATABASE_NAME` is defined, # it returns ":memory:". db_name = connection.creation._create_test_db(verbosity, autoclobber) # Closing out the current connection to the database set in # originally in the settings. This makes it so `initialize_spatialite` # function will be run on the connection for the _test_ database instead. connection.close() # Point to the new database settings.DATABASE_NAME = db_name connection.settings_dict["DATABASE_NAME"] = db_name can_rollback = connection.creation._rollback_works() settings.DATABASE_SUPPORTS_TRANSACTIONS = can_rollback connection.settings_dict["DATABASE_SUPPORTS_TRANSACTIONS"] = can_rollback # Finally, loading up the SpatiaLite SQL file. load_spatialite_sql(db_name, verbosity=verbosity) if verbosity >= 1: print 'Creation of spatial database %s successful.' % db_name # Syncing the database call_command('syncdb', verbosity=verbosity, interactive=interactive)
def _run(plugin): plugin.run_check() try: return plugin finally: from django.db import connection connection.close()
def handle(self, *args, **options): """Django command handler.""" self.verbosity = int(options.get('verbosity')) self.quiet = options.get('quiet') try: connection.close() self.filepath = options.get('filepath') self.backup_extension = options.get('backup_extension') or 'backup' self.servername = options.get('servername') self.decrypt = options.get('decrypt') self.uncompress = options.get('uncompress') self.passphrase = options.get('passphrase') self.interactive = options.get('interactive') self.database = self._get_database(options) self.storage = BaseStorage.storage_factory() self.database = self._get_database(options) if 'mongo' in self.database['ENGINE']: self.dbcommands = MongoDBCommands(self.database) else: self.dbcommands = DBCommands(self.database) if not self.backup_extension: self.backup_extension = self.dbcommands.settings.extension or 'backup' if options.get('list'): return self.list_backups() self.restore_backup() except StorageError as err: raise CommandError(err)
def day_wise_logins(request): if request.user.is_superuser: from django.db import connection cursor = connection.cursor() cursor.execute("select count(*) from auth_user where datediff(date(now()),date(last_login)) > 7;") inactive_users_week = cursor.fetchall()[0][0] cursor.execute("select count(*) from auth_user where datediff(date(now()),date(last_login)) > 14;") inactive_users_2week = cursor.fetchall()[0][0] cursor.execute("select count(*) from auth_user where datediff(date(now()),date(last_login)) > 21;") inactive_users_3week = cursor.fetchall()[0][0] cursor.execute("select count(*) from auth_user where datediff(date(now()),date(last_login)) > 28;") inactive_users_4week = cursor.fetchall()[0][0] cursor.execute("select count(*) from auth_user where datediff(date(now()),date(last_login)) > 56;") inactive_users_8week = cursor.fetchall()[0][0] cursor.execute("select count(*) from auth_user where datediff(date(now()),date(last_login)) > 84;") inactive_users_12week = cursor.fetchall()[0][0] cursor.execute("select count(*) from auth_user where last_login=date_joined and is_active=1;") one_time_loggers = cursor.fetchall()[0][0] cursor.execute("select count(*) from auth_user where date(last_login)=date(date_joined) and is_active=1;") one_day_loggers = cursor.fetchall()[0][0] connection.close() return render_to_response("console/daily_logins.html", {'one_time_loggers':one_time_loggers,'one_day_loggers':one_day_loggers,'inactive_users_week':inactive_users_week,'inactive_users_2week': inactive_users_2week,'inactive_users_3week':inactive_users_3week, 'inactive_users_4week':inactive_users_4week, 'inactive_users_8week':inactive_users_8week,'inactive_users_12week':inactive_users_12week}) else: return HttpResponseRedirect('/'+request.user.username+'/')
def create_test_spatial_db(verbosity=1, autoclobber=False, interactive=False): "Creates a test spatial database based on the settings." # Making sure we're using PostgreSQL and psycopg2 if settings.DATABASE_ENGINE != "postgresql_psycopg2": raise Exception("Spatial database creation only supported postgresql_psycopg2 platform.") # Getting the spatial database name db_name = get_spatial_db(test=True) _create_with_cursor(db_name, verbosity=verbosity, autoclobber=autoclobber) # If a template database is used, then don't need to do any of the following. if not hasattr(settings, "POSTGIS_TEMPLATE"): # Creating the db language, does not need to be done on NT platforms # since the PostGIS installer enables this capability. if os.name != "nt": create_lang(db_name, verbosity=verbosity) # Now adding in the PostGIS routines. load_postgis_sql(db_name, verbosity=verbosity) if verbosity >= 1: print "Creation of spatial database %s successful." % db_name # Closing the connection connection.close() settings.DATABASE_NAME = db_name connection.settings_dict["DATABASE_NAME"] = db_name can_rollback = connection.creation._rollback_works() settings.DATABASE_SUPPORTS_TRANSACTIONS = can_rollback connection.settings_dict["DATABASE_SUPPORTS_TRANSACTIONS"] = can_rollback # Syncing the database call_command("syncdb", verbosity=verbosity, interactive=interactive)
def _send_bulk(emails, uses_multiprocessing=True, log_level=None): # Multiprocessing does not play well with database connection # Fix: Close connections on forking process # https://groups.google.com/forum/#!topic/django-users/eCAIY9DAfG0 if uses_multiprocessing: db_connection.close() if log_level is None: log_level = get_log_level() sent_count, failed_count = 0, 0 email_count = len(emails) logger.info('Process started, sending %s emails' % email_count) try: for email in emails: status = email.dispatch(log_level=log_level, disconnect_after_delivery=False) if status == STATUS.sent: sent_count += 1 logger.debug('Successfully sent email #%d' % email.id) else: failed_count += 1 logger.debug('Failed to send email #%d' % email.id) except Exception as e: logger.error(e, exc_info=sys.exc_info(), extra={'status_code': 500}) connections.close() logger.info('Process finished, %s attempted, %s sent, %s failed' % (email_count, sent_count, failed_count)) return (sent_count, failed_count)
def feature_new_users(request): # criteria for featuring would be on desc order of comments, except team but comment count > 100 from django.db import connection cursor = connection.cursor() cursor.execute("select count(*) as cnt ,user_id from django_comments where user_id not in (3,4,6,53,1571,465) group by user_id order by cnt desc limit 50;") active_users = cursor.fetchall() connection.close() featured_users = Featured_User.objects.all() count = 3 now = datetime.datetime.now() date = now.date()-datetime.timedelta(99) for user in active_users: #comment count should be greater than 100 if int(user[0])>15 and count > 0: usr = get_object_or_404(User,id=int(user[1])) if not Featured_User.objects.filter(user=usr): feat_user = Featured_User(user = usr) feat_user.save() count = count -1 params_for_mail = {'#_1':usr.username} send_mail(str(usr.email),'Kwippy <*****@*****.**>','featured_user',params_for_mail) #delete featured users who were featured 99 days back so that they can be featured again. old_feat_users = Featured_User.objects.filter(created_at__lte=date) old_feat_users.delete() featured_users = Featured_User.objects.all() return render_to_response("console/featured_users.html",{'featured_users':featured_users})
def create_products(queue): """ Download an image from Flickr for the product on the queue and if successful now or previously, create the applicable product records. """ # Close the connection for this process to avoid the issue discussed here: # http://groups.google.com/group/django-users/ # browse_thread/thread/2c7421cdb9b99e48 connection.close() product_options = ProductOption.objects.as_fields() while True: # Get next set of data from queue. data = queue.get() if data is None: break main_category, sub_category, product = data[0], data[1], data[-1] # Try and download a product image from Flickr. image = join(image_dir, "%s.jpg" % product) if exists(image): message = "Using already downloaded image for %s" % data else: try: images = flickr.photos_search(tags=[product], per_page=1) if not images: raise Exception("No images found") url = images[0].getURL(size="Large", urlType="source") urlretrieve(url, image) except Exception, e: message = "Error [%s] for %s" % (e, data) else: message = "Successfully downloaded image for %s" % data
def create_spatial_db(test=False, verbosity=1, autoclobber=False, interactive=False): "Creates a spatial database based on the settings." # Making sure we're using PostgreSQL and psycopg2 if settings.DATABASE_ENGINE != 'postgresql_psycopg2': raise Exception('Spatial database creation only supported postgresql_psycopg2 platform.') # Getting the spatial database name if test: db_name = get_spatial_db(test=True) _create_with_cursor(db_name, verbosity=verbosity, autoclobber=autoclobber) else: db_name = get_spatial_db() _create_with_shell(db_name, verbosity=verbosity, autoclobber=autoclobber) # Creating the db language, does not need to be done on NT platforms # since the PostGIS installer enables this capability. if os.name != 'nt': create_lang(db_name, verbosity=verbosity) # Now adding in the PostGIS routines. load_postgis_sql(db_name, verbosity=verbosity) if verbosity >= 1: print 'Creation of spatial database %s successful.' % db_name # Closing the connection connection.close() settings.DATABASE_NAME = db_name # Syncing the database call_command('syncdb', verbosity=verbosity, interactive=interactive)
def loadCart(username): cursor = connection.cursor() sql = "SELECT id FROM auth_user WHERE username = %s" user_id = cursor.execute(sql,[username]).fetchone()[0] sql = "SELECT cart_id FROM cart WHERE user_id = %s" cart_id = cursor.execute(sql,[user_id]).fetchone() if cart_id is None : return 0 else: cart_id = cart_id[0] sql = "SELECT product_id,name,price,image FROM product " \ "WHERE product_id in " \ "(SELECT product_id FROM cart_item WHERE cart_id = %s)" re = cursor.execute(sql,[cart_id]).fetchall() data = [] i = 0 for row in re: sql = "SELECT amount FROM cart_item WHERE cart_id = %s AND product_id = %s" amount = cursor.execute(sql,[cart_id,row[0]]).fetchone()[0] sql = "SELECT stock_amount FROM product WHERE product_id = %s" cur_amount = cursor.execute(sql,[row[0]]).fetchone()[0] if amount > cur_amount: amount = cur_amount data.append(str(row[0]) + SPLIT_SBAR + row[1] + SPLIT_SBAR + str(row[2]) + SPLIT_SBAR + row[3] + SPLIT_SBAR + str(amount) + SPLIT_LBAR) i = i + 1 data.append('&' + str(i)) connection.close() return data
def do_db(self, callback, **args): try: return callback(**args) except (OperationalError, InterfaceError) as e: # Connection has gone away self.logger.warn("%s, reconnecting" % e) # TODO connection.close() return callback(**args)
def run(self): while True: try: if self.exit: return if self.paused: self.sleep() continue if not self.last_check: if not self.queue_running(): self.sleep() continue else: diff = timezone.now() - self.last_check if diff.total_seconds() > 30: if not self.queue_running(): self.sleep() continue self.assign_download() self.check_finished() self.extract() except OperationalError as e: connection.close() logger.info("Resetting mysql connection due to %s" % str(e)) except Exception as e: logger.exception("Some error in main thread %s" % str(e)) self.sleep()
def run(self): while True: if self.abort: return last_run = cache.get("maintenance_run") if None is last_run: self.do_maintenance() continue now = datetime.now() diff = now - last_run hours = diff.total_seconds() / 60 / 60 if hours >= 24: try: self.do_maintenance() except OperationalError as e: connection.close() logger.info("Resetting mysql connection due to %s" % str(e)) except Exception as e: logger.exception("Error in extractor thread %s" % str(e)) self.sleep()
def _send_messages(email_messages): connection = _get_real_backend() # Create a messages on a database for correct # tracking of their status. email_models = [models.Message.from_email_message(email, save=True) for email in email_messages] # Open connection for send all messages connection.open() sended_counter = 0 for email, model_instance in zip(email_messages, email_models): if hasattr(email, "priority"): if email.priority <= models.PRIORITY_LOW: model_instance.priority = email.priority model_instance.status = models.STATUS_PENDING model_instance.save() continue sended = connection.send_messages([email]) if sended == 1: sended_counter += 1 model_instance.status = models.STATUS_SENT model_instance.sent_at = timezone.now() else: model_instance.status = models.STATUS_FAILED model_instance.save() connection.close() return sended_counter
def _retry_send_messages(): """ Function that retry send failed messages. """ max_retry_value = getattr(settings, "DJMAIL_MAX_RETRY_NUMBER", 3) queryset = models.Message.objects.filter(status=models.STATUS_FAILED)\ .filter(retry_count__lte=max_retry_value)\ .order_by("-priority", "created_at") connection = _get_real_backend() paginator = Paginator(list(queryset), getattr(settings, "DJMAIL_MAX_BULK_RETRY_SEND", 10)) for page_index in paginator.page_range: connection.open() for message_model in paginator.page(page_index).object_list: email = message_model.get_email_message() sended = connection.send_messages([email]) if sended == 1: message_model.status = models.STATUS_SENT message_model.sent_at = timezone.now() else: message_model.retry_count += 1 message_model.save() connection.close()
def get_count(request, model, refresh, processor, context): opts = model._meta # Build a queryset through the context which is toggled by # the parameter. processor = processor(context=context, tree=model) queryset = processor.get_queryset(request=request) # Get count from cache or database label = ':'.join([opts.app_label, opts.module_name, 'count']) key = cache_key(label, kwargs={'queryset': queryset}) cache = get_cache(avocado_settings.DATA_CACHE) if refresh: count = None else: count = cache.get(key) if count is None: count = queryset.values('pk').distinct().count() cache.set(key, count, timeout=NEVER_EXPIRE) # Close the connection in the thread to prevent 'idle in transaction' # situtations. from django.db import connection connection.close() return count
def addToFav(user_id,pid): cursor = connection.cursor() connection.close() if checkFavExist(user_id) is True: updateFav(user_id,pid) else: insertFav(user_id,pid)
def set_distribution(ldamodel): """Transformacion para dejar los documentos como listas de los word id""" insert = """INSERT INTO application_documentdistribution (document_id, ldamodel_id, distribution) ( SELECT D.id, SL.ldamodel_id, group_concat(REPEAT(CONCAT(HEX(F.word_id),','),F.frequency) separator '') as distribution FROM application_frequency F JOIN application_wordldamodel W ON F.word_id = W.word_id AND W.ldamodel_id = """+str(ldamodel.id)+""" JOIN application_document D ON F.document_id = D.id JOIN application_dataset S ON S.id = D.dataset_id JOIN application_datasetldamodel SL ON S.id = SL.dataset_id WHERE SL.ldamodel_id = """+str(ldamodel.id)+""" GROUP BY D.id );""" print insert cursor = connection.cursor() cursor.execute("SET GLOBAL group_concat_max_len = 5000000;") cursor.execute("COMMIT") cursor.execute("DELETE FROM application_documentdistribution WHERE ldamodel_id = %s" % str(ldamodel.id)) cursor.execute("COMMIT") cursor.execute(insert) cursor.execute("COMMIT") connection.close() print " Distribuciones Asignadas..." return HttpResponse("Set distribution completed")
def perform_work(self, body): try: flush = body.get('event') == 'FLUSH' if flush: self.last_event = '' if not flush: event_map = { 'job_id': JobEvent, 'ad_hoc_command_id': AdHocCommandEvent, 'project_update_id': ProjectUpdateEvent, 'inventory_update_id': InventoryUpdateEvent, 'system_job_id': SystemJobEvent, } job_identifier = 'unknown job' for key, cls in event_map.items(): if key in body: job_identifier = body[key] break self.last_event = f'\n\t- {cls.__name__} for #{job_identifier} ({body.get("event", "")} {body.get("uuid", "")})' # noqa if body.get('event') == 'EOF': try: final_counter = body.get('final_counter', 0) logger.info( 'Event processing is finished for Job {}, sending notifications' .format(job_identifier)) # EOF events are sent when stdout for the running task is # closed. don't actually persist them to the database; we # just use them to report `summary` websocket events as an # approximation for when a job is "done" emit_channel_notification( 'jobs-summary', dict(group_name='jobs', unified_job_id=job_identifier, final_counter=final_counter)) # Additionally, when we've processed all events, we should # have all the data we need to send out success/failure # notification templates uj = UnifiedJob.objects.get(pk=job_identifier) if isinstance(uj, Job): # *actual playbooks* send their success/failure # notifications in response to the playbook_on_stats # event handling code in main.models.events pass elif hasattr(uj, 'send_notification_templates'): handle_success_and_failure_notifications.apply_async( [uj.id]) except Exception: logger.exception( 'Worker failed to emit notifications: Job {}'. format(job_identifier)) return event = cls.create_from_data(**body) self.buff.setdefault(cls, []).append(event) retries = 0 while retries <= self.MAX_RETRIES: try: self.flush(force=flush) break except (OperationalError, InterfaceError, InternalError): if retries >= self.MAX_RETRIES: logger.exception( 'Worker could not re-establish database connectivity, giving up on one or more events.' ) return delay = 60 * retries logger.exception( 'Database Error Saving Job Event, retry #{i} in {delay} seconds:' .format(i=retries + 1, delay=delay)) django_connection.close() time.sleep(delay) retries += 1 except DatabaseError: logger.exception('Database Error Saving Job Event') break except Exception as exc: tb = traceback.format_exc() logger.error('Callback Task Processor Raised Exception: %r', exc) logger.error('Detail: {}'.format(tb))
def update_birthday_slowly(): Person.objects.update_or_create( first_name='John', defaults={'birthday': birthday_sleep}) # Avoid leaking connection for Oracle connection.close()
def _send_bulk(emails, uses_multiprocessing=True, log_level=None): # Multiprocessing does not play well with database connection # Fix: Close connections on forking process # https://groups.google.com/forum/#!topic/django-users/eCAIY9DAfG0 if uses_multiprocessing: db_connection.close() if log_level is None: log_level = get_log_level() sent_emails = [] failed_emails = [] # This is a list of two tuples (email, exception) email_count = len(emails) logger.info('Process started, sending %s emails' % email_count) def send(email): try: email.dispatch(log_level=log_level, commit=False) sent_emails.append(email) logger.debug('Successfully sent email #%d' % email.id) except Exception as e: logger.debug('Failed to send email #%d' % email.id) failed_emails.append((email, e)) # Prepare emails before we send these to threads for sending # So we don't need to access the DB from within threads for email in emails: # Sometimes this can fail, for example when trying to render # email from a faulty Django template try: email.prepare_email_message() except Exception as e: failed_emails.append((email, e)) number_of_threads = min(get_threads_per_process(), email_count) pool = ThreadPool(number_of_threads) pool.map(send, emails) pool.close() pool.join() connections.close() # Update statuses of sent and failed emails email_ids = [email.id for email in sent_emails] OutgoingEmail.objects.filter(id__in=email_ids).update(status=STATUS.sent) email_ids = [email.id for (email, e) in failed_emails] OutgoingEmail.objects.filter(id__in=email_ids).update(status=STATUS.failed) # If log level is 0, log nothing, 1 logs only sending failures # and 2 means log both successes and failures if log_level >= 1: logs = [] for (email, exception) in failed_emails: logs.append( Log(email=email, status=STATUS.failed, message=str(exception), exception_type=type(exception).__name__) ) if logs: Log.objects.bulk_create(logs) if log_level == 2: logs = [] for email in sent_emails: logs.append(Log(email=email, status=STATUS.sent)) if logs: Log.objects.bulk_create(logs) logger.info( 'Process finished, %s attempted, %s sent, %s failed' % ( email_count, len(sent_emails), len(failed_emails) ) ) return len(sent_emails), len(failed_emails)
def release_connection(): """ Close thread's conneciton to database and increment database resource count. """ connection.close() conn_sem.release()
def main_loop(self, options): last_schedule = last_dispatcher_check = time.time() while True: try: try: # Compute the timeout now = time.time() timeout = min( SCHEDULE_INTERVAL - (now - last_schedule), PING_INTERVAL - (now - last_dispatcher_check), ) # If some actions are remaining, decrease the timeout if any([self.events[k] for k in self.events.keys()]): timeout = min(timeout, 2) # Wait at least for 1ms timeout = max(timeout * 1000, 1) # Wait for data or a timeout sockets = dict(self.poller.poll(timeout)) except zmq.error.ZMQError: continue if sockets.get(self.pipe_r) == zmq.POLLIN: self.logger.info("[POLL] Received a signal, leaving") break # Command socket if sockets.get(self.controler) == zmq.POLLIN: while self.controler_socket( ): # Unqueue all pending messages pass # Events socket if sockets.get(self.event_socket) == zmq.POLLIN: self.logger.info("[EVENT] handling events") while self.read_event_socket( ): # Unqueue all pending messages pass # Wait for the next iteration to handle the event. # In fact, the code that generated the event (lava-logs or # lava-server-gunicorn) needs some time to commit the # database transaction. # If we are too fast, the database object won't be # available (or in the right state) yet. continue # Inotify socket if sockets.get(self.inotify_fd) == zmq.POLLIN: os.read(self.inotify_fd, 4096) if self.auth is not None: self.logger.info( "[AUTH] Reloading certificates from %s", options["slaves_certs"], ) self.auth.configure_curve( domain="*", location=options["slaves_certs"]) else: self.logger.error( "[AUTH] New certificates in %s but encryption is disabled", options["slaves_certs"], ) # Check dispatchers status now = time.time() if now - last_dispatcher_check > PING_INTERVAL: for hostname, dispatcher in self.dispatchers.items(): if (dispatcher.online and now - dispatcher.last_msg > DISPATCHER_TIMEOUT): if hostname == "lava-logs": self.logger.error( "[STATE] lava-logs goes OFFLINE") else: self.logger.error( "[STATE] Dispatcher <%s> goes OFFLINE", hostname) self.dispatchers[hostname].go_offline() last_dispatcher_check = now # Limit accesses to the database. This will also limit the rate of # CANCEL and START messages if time.time() - last_schedule > SCHEDULE_INTERVAL: if self.dispatchers["lava-logs"].online: schedule(self.logger) # Dispatch scheduled jobs with transaction.atomic(): self.start_jobs() else: self.logger.warning( "lava-logs is offline: can't schedule jobs") # Handle canceling jobs with transaction.atomic(): self.cancel_jobs() # Do not count the time taken to schedule jobs last_schedule = time.time() else: # Cancel the jobs and remove the jobs from the set if self.events["canceling"]: with transaction.atomic(): self.cancel_jobs(partial=True) self.events["canceling"] = set() # Schedule for available device-types if self.events["available_dt"]: jobs = schedule(self.logger, self.events["available_dt"]) self.events["available_dt"] = set() # Dispatch scheduled jobs with transaction.atomic(): self.start_jobs(jobs) except (OperationalError, InterfaceError): self.logger.info("[RESET] database connection reset.") # Closing the database connection will force Django to reopen # the connection connection.close() time.sleep(2)
def query(request): instance_name = request.POST.get('instance_name') sql_content = request.POST.get('sql_content') db_name = request.POST.get('db_name') limit_num = request.POST.get('limit_num') result = {'status': 0, 'msg': 'ok', 'data': {}} try: instance = Instance.objects.get(instance_name=instance_name) except Instance.DoesNotExist: result['status'] = 1 result['msg'] = '实例不存在' return result # 服务器端参数验证 if sql_content is None or db_name is None or instance_name is None or limit_num is None: result['status'] = 1 result['msg'] = '页面提交参数可能为空' return HttpResponse(json.dumps(result), content_type='application/json') sql_content = sql_content.strip() # 获取用户信息 user = request.user # 过滤注释语句和非查询的语句 sql_content = ''.join( map( lambda x: re.compile(r'(^--\s+.*|^/\*.*\*/;\s*$)').sub( '', x, count=1), sql_content.splitlines(1))).strip() # 去除空行 sql_content = re.sub('[\r\n\f]{2,}', '\n', sql_content) # 语法判断 sql_list = sqlparse.split(sql_content) for sql in sql_list: if re.match(r"^select|^show|^explain", sql.lower()): break else: result['status'] = 1 result['msg'] = '仅支持^select|^show|^explain语法,请联系管理员!' return HttpResponse(json.dumps(result), content_type='application/json') # 执行第一条有效sql sql_content = sql_list[0].rstrip(';') try: # 查询权限校验 priv_check_info = query_priv_check(user, instance_name, db_name, sql_content, limit_num) if priv_check_info['status'] == 0: limit_num = priv_check_info['data']['limit_num'] priv_check = priv_check_info['data']['priv_check'] else: return HttpResponse(json.dumps(priv_check_info), content_type='application/json') limit_num = 0 if re.match(r"^explain", sql_content.lower()) else limit_num # 查询检查 query_engine = get_engine(instance=instance) filter_result = query_engine.query_check(db_name=db_name, sql=sql_content, limit_num=limit_num) if filter_result.get( 'bad_query') and SysConfig().get('disable_star') is True: result['status'] = 1 result['msg'] = filter_result.get('msg') return HttpResponse(json.dumps(result), content_type='application/json') else: sql_content = filter_result['filtered_sql'] sql_content = sql_content + ';' # 执行查询语句,统计执行时间 t_start = time.time() query_result = query_engine.query(db_name=str(db_name), sql=sql_content, limit_num=limit_num) t_end = time.time() query_result.query_time = "%5s" % "{:.4f}".format(t_end - t_start) # 数据脱敏,同样需要检查配置,是否开启脱敏,语法树解析是否允许出错继续执行 hit_rule = 0 if re.match(r"^select", sql_content.lower( )) else 2 # 查询是否命中脱敏规则,0, '未知', 1, '命中', 2, '未命中' masking = 2 # 查询结果是否正常脱敏,1, '是', 2, '否' t_start = time.time() # 仅对正确查询的语句进行脱敏 if SysConfig().get('data_masking') and re.match( r"^select", sql_content.lower()) and query_result.error is None: try: query_result = query_engine.query_masking( db_name=db_name, sql=sql_content, resultset=query_result) if SysConfig().get( 'query_check') and query_result.is_critical is True: masking_result = { 'status': query_result.status, 'msg': query_result.error, 'data': query_result.__dict__ } return HttpResponse(json.dumps(masking_result), content_type='application/json') else: # 实际未命中, 则显示为未做脱敏 if query_result.is_masked: masking = 1 hit_rule = 1 except Exception: logger.error(traceback.format_exc()) # 报错, 未脱敏, 未命中 hit_rule = 2 masking = 2 if SysConfig().get('query_check'): result['status'] = 1 result['msg'] = '脱敏数据报错,请联系管理员' return HttpResponse(json.dumps(result), content_type='application/json') t_end = time.time() query_result.mask_time = "%5s" % "{:.4f}".format(t_end - t_start) sql_result = query_result.__dict__ result['data'] = sql_result # 成功的查询语句记录存入数据库 if sql_result.get('error'): pass else: query_log = QueryLog() query_log.username = user.username query_log.user_display = user.display query_log.db_name = db_name query_log.instance_name = instance_name query_log.sqllog = sql_content if int(limit_num) == 0: limit_num = int(sql_result['affected_rows']) else: limit_num = min(int(limit_num), int(sql_result['affected_rows'])) query_log.effect_row = limit_num query_log.cost_time = query_result.query_time query_log.priv_check = priv_check query_log.hit_rule = hit_rule query_log.masking = masking # 防止查询超时 try: query_log.save() except: connection.close() query_log.save() except Exception as e: logger.error(traceback.format_exc()) result['status'] = 1 result['msg'] = str(e) # 返回查询结果 try: return HttpResponse(json.dumps(result, cls=ExtendJSONEncoder, bigint_as_string=True), content_type='application/json') except Exception: return HttpResponse(json.dumps(result, default=str, bigint_as_string=True, encoding='latin1'), content_type='application/json')
def wrapper(*args, **kwagrs): try: connection.connection.ping() except Exception: connection.close() return func(*args, **kwagrs)
def execute_workflow(self, manual=False): """执行上线单""" workflow_detail = self.workflow if workflow_detail.is_manual == 1: return self.execute(db_name=workflow_detail.db_name, sql=workflow_detail.sql_content) execute_result = ReviewSet(full_sql=workflow_detail.sql_content) inception_engine = InceptionEngine() if workflow_detail.is_backup == '是': str_backup = "--enable-remote-backup;" else: str_backup = "--disable-remote-backup;" # 根据inception的要求,执行之前最好先split一下 sql_split = "/*--user=%s; --password=%s; --host=%s; --enable-execute;--port=%d; --enable-ignore-warnings;--enable-split;*/\ inception_magic_start;\ use %s;\ %s\ inception_magic_commit;" % ( self.user, self.password, self.host, self.port, workflow_detail.db_name, workflow_detail.sql_content) split_result = inception_engine.query(sql=sql_split) execute_result.rows = [] # 对于split好的结果,再次交给inception执行.这里无需保持在长连接里执行,短连接即可. for splitRow in split_result.rows: sql_tmp = splitRow[1] sql_execute = "/*--user=%s;--password=%s;--host=%s;--enable-execute;--port=%d; --enable-ignore-warnings;%s*/\ inception_magic_start;\ %s\ inception_magic_commit;" % ( self.user, self.password, self.host, self.port, str_backup, sql_tmp) one_line_execute_result = inception_engine.query(sql=sql_execute) # 执行, 把结果转换为ReviewSet for sqlRow in one_line_execute_result.to_dict(): execute_result.rows.append(ReviewResult( id=sqlRow['ID'], stage=sqlRow['stage'], errlevel=sqlRow['errlevel'], stagestatus=sqlRow['stagestatus'], errormessage=sqlRow['errormessage'], sql=sqlRow['SQL'], affected_rows=sqlRow['Affected_rows'], actual_affected_rows=sqlRow['Affected_rows'], sequence=sqlRow['sequence'], backup_dbname=sqlRow['backup_dbname'], execute_time=sqlRow['execute_time'], sqlsha1=sqlRow['sqlsha1'])) # 每执行一次,就将执行结果更新到工单的execute_result workflow_detail.execute_result = execute_result.json() from django.db import connection if connection.connection is not None: connection.close() workflow_detail.save() # 二次加工一下,目的是为了和sqlautoReview()函数的return保持格式一致,便于在detail页面渲染. execute_result.status = "已正常结束" for sqlRow in execute_result.rows: # 如果发现任何一个行执行结果里有errLevel为1或2,并且stagestatus列没有包含Execute Successfully字样,则判断最终执行结果为有异常. if (sqlRow.errlevel == 1 or sqlRow.errlevel == 2) and re.match(r"\w*Execute Successfully\w*", sqlRow.stagestatus) is None: execute_result.status = "执行有异常" execute_result.error = "Line {0} has error/warning: {1}".format(sqlRow.id, sqlRow.errormessage) return execute_result
def stationSearch(request): type = request.POST['type'] keyword = request.POST['keyword'] # print("Check Post -", type, keyword) print("Check Post -", type, keyword) if type == 'statnm': try: cursor = connection.cursor() strSql = "select evst.statNm,evst.addr,evst.lat,evst.lng,evst.useTime,evst.busicall,descInfo,congestion " \ "from ev_station evst \ join ev_real_time evtm on(evst.evsn=evtm.evsn),(select c.evSn, group_concat(des SEPARATOR '\n') as descInfo " \ "from (select a.evSn, a.chgerId, concat('기기 번호 :', a.chgerId , ' ( 상태 : ' , (select codeName from ev_code_inf where codeId = a.stat) , ', 충전타입 : ' , GROUP_CONCAT((select codeName from ev_code_inf where codeId = b.chgerType) SEPARATOR ','),')') as des " \ "from ev.ev_station_status a, ev.ev_station_chgertype b " \ "where a.evSn = b.evSn group by a.evSn, a.chgerId) c group by c.evSn) info " \ "where evst.evSn = info.evSn and evst.statNm Like '%" + keyword + "%';" result = cursor.execute(strSql) stations = cursor.fetchall() print('stations - ', stations) connection.commit() connection.close() except: connection.rollback() print('Failed selecting in stations') elif type == 'addr': try: cursor = connection.cursor() strSql = "select evst.statNm,evst.addr,evst.lat,evst.lng,evst.useTime,evst.busicall, info.descInfo " \ "from ev.ev_station evst,(select c.evSn, group_concat(des SEPARATOR '\n') as descInfo " \ "from (select a.evSn, a.chgerId, concat('기기 번호 :', a.chgerId , ' ( 상태 : ' , (select codeName from ev_code_inf where codeId = a.stat) , ', 충전타입 : ' , GROUP_CONCAT((select codeName from ev_code_inf where codeId = b.chgerType) SEPARATOR ','),')') as des " \ "from ev.ev_station_status a, ev.ev_station_chgertype b " \ "where a.evSn = b.evSn group by a.evSn, a.chgerId) c group by c.evSn) info " \ "where evst.evSn = info.evSn and evst.addr Like '%" + keyword + "%';" result = cursor.execute(strSql) stations = cursor.fetchall() print('stations - ', stations) connection.commit() connection.close() except: connection.rollback() print('Failed selecting in stations') list = [] cnt = 0 for station in stations: row = {'statNm': station[0], 'addr': station[1], 'lat': station[2], 'lng': station[3], 'useTime': station[4], 'busiCall': station[5], 'descInfo': station[6], 'congestion': station[7], } list.append(row) cnt += 1 if cnt == 100: break for a in list: print("check - ", a) print(len(list)) return JsonResponse(list, safe=False)
def post(self, request): res = {'status': False} option = request.GET.get('option', None) if option: if option == 'add': addr = request.POST.get('addr', None) named = request.POST.get('named', None) newer = request.POST.get('newer', None) nper = request.POST.get('nper', None) time_rule = request.POST.get('time_rule', None) first_status = request.POST.get('first_status', None) if danger.xss(named): return JsonResponse({'status': False, 'msg': 'xss'}) if (newer == 'true') or (newer == True): contact = user_modles.Contact() contact.email = addr contact.first_named = named contact.save() else: contact = user_modles.Contact.objects.filter( Q(email=addr) & Q(status=True)) contact = contact[0] visit_time = request.POST.get('visit_time', None) email_template = request.POST.get('email_template', None) ea = models.EmailApply() if (first_status == 'false') or (first_status == False): ea.first_status = False ea.nper = nper ea.now_time_rule = time_rule ea.contact = user_modles.Contact.objects.get(id=contact.id) ea.visit_time = visit_time ea.email_template = models.EmailTemplate.objects.get( id=int(email_template)) ea.save() connection.close() worker = APSTask.TaskProcess([ea.id], common.WAY[1][0]) worker.start() res['id'] = ea.id res['status'] = True if option == 'tasker': numed = settings.EVERY tasks = [] index = 0 ids = request.POST.get('ids', None) ids = ids.split(',') nper = request.POST.get('nper', None) time_rule = request.POST.get('time_rule', None) visit_time = request.POST.get('visit_time', None) email_template = request.POST.get('email_template', None) first_status = request.POST.get('first_status', None) if (first_status == 'false') or (first_status == False): first_status = False else: first_status = True for e in ids: contact = user_modles.Contact.objects.filter( Q(id=e) & Q(status=True)) if contact: contact = contact[0] ea = models.EmailApply() ea.nper = nper ea.now_time_rule = time_rule ea.contact = contact ea.visit_time = visit_time ea.email_template = models.EmailTemplate.objects.get( id=int(email_template)) ea.first_status = first_status ea.save() tasks.append(str(ea.id)) if len(tasks) % 5 == 0: time.sleep(0.5) index = len(tasks) tasks = [ tasks[i:i + numed] for i in range(0, len(tasks), numed) ] for ts in tasks: ts = '_'.join(ts) running = web_models.Running() running.way = common.WAY[1][0] running.ids = ts running.done_status = False running.block_status = False running.save() res['status'] = True res['numed'] = index if option == 'update': pk = request.POST.get('id', None) if pk is None: return JsonResponse(res) send_status = request.POST.get('send_status', None) visit_time = request.POST.get('visit_time', None) if int(send_status) == 1: send_status = False else: send_status = True ea = models.EmailApply.objects.get(id=pk) ea.visit_time = visit_time ea.send_status = send_status ea.save() res['status'] = True res['instance'] = { 'visit_time': visit_time, 'send_status': send_status } if option == 'trash': pk = request.GET.get('id', None) if pk is None: pk = request.POST.get('id', None) ea = models.EmailApply.objects.get(id=pk) ea.status = False ea.save() res['status'] = True res['instance'] = { 'named': ea.contact.first_named, 'addr': ea.contact.email, 'service': ea.email_template.service } return JsonResponse(res)
def ExecMakeTarget(storemedia, targetvguuid, targetHost, clientiqn, serviceName, storageSize, aagroup, clumpgroup, subnet, owner): chosenVG = VG.objects.get(vguuid=targetvguuid) clientiqnHash = hashlib.sha1(clientiqn).hexdigest()[:8] iqnTarget = "".join( ["iqn.2014.01.", targetHost, ":", serviceName, ":", clientiqnHash]) try: targets = Target.objects.filter( iqntar__contains="".join([serviceName, ":", clientiqnHash])) if len(targets) == 0: raise ObjectDoesNotExist for t in targets: iqnComponents = t.iqntar.split(':') if ((serviceName == iqnComponents[1]) and (clientiqnHash == iqnComponents[2])): logger.info( 'Target already exists for (serviceName=%s,clientiqn=%s) tuple' % (serviceName, clientiqn)) try: existingTargetstoremedia = LV.objects.get( target=t).vg.storemedia except: logger.error( "Target %s exists in DB but LV does not, inconsistent" % (t.iqntar)) return ( -1, "Target %s exists in DB but LV does not, inconsistent" % (t.iqntar)) if (existingTargetstoremedia == storemedia): return (1, t.iqntar) else: errorStr = "Target %s on DIFFERENT storemedia %s already exists." % ( t.iqntar, existingTargetstoremedia) logger.info(errorStr) return (-1, errorStr) else: raise ObjectDoesNotExist except ObjectDoesNotExist: # try: # if subnet != 'public': # IPRange.objects.get(iprange=subnet) # except: # logger.debug('Subnet %s not found on host %s while trying to create target %s, creation aborted, contact admin' %(subnet, targetHost, iqnTarget )) # return (-1,"Invalid subnet specified") (quotaFlag, quotaReason) = CheckUserQuotas(float(storageSize), owner) if quotaFlag == -1: logger.debug(quotaReason) return (-1, quotaReason) else: logger.info(quotaReason) logger.info( "Creating new target for request {%s %s %s}, this is the generated iSCSItarget: %s" % (clientiqn, serviceName, str(storageSize), iqnTarget)) targethost = StorageHost.objects.get(dnsname=targetHost) p = PollServer(targetHost) storeip1 = targethost.storageip1 storeip2 = targethost.storageip2 if subnet != 'public': try: storeip1 = Interface.objects.get( owner=owner, storagehost=targethost, iprange__iprange=unicode(subnet)).ip storeip2 = storeip1 except: logger.error( 'Chosen host %s is missing IP addresses in requested subnet' % (targethost, )) return ( -1, 'Error in host network configuration or ownership for the required subnet, contact storage admin' ) if p.CreateTarget(iqnTarget, clientiqn, str(storageSize), storeip1, storeip2, targetvguuid) == 1: logger.info("SUCCESSFUL TARGET RUN") BASE_DIR = os.path.dirname(os.path.dirname(__file__)) config = ConfigParser.RawConfigParser() config.read(os.path.join(BASE_DIR, 'saturn.ini')) (devDic, tarDic) = ParseSCSTConf( os.path.join(BASE_DIR, config.get('saturnring', 'iscsiconfigdir'), targetHost + '.scst.conf')) logger.info("DevDic = " + str(devDic)) logger.info("TarDic = " + str(tarDic)) if iqnTarget in tarDic: newTarget = Target(owner=owner, targethost=targethost, iqnini=clientiqn, iqntar=iqnTarget, sizeinGB=float(storageSize), storageip1=storeip1, storageip2=storeip2) newTarget.save() lvDict = p.GetLVs(targetvguuid) lvName = 'lvol-' + hashlib.md5(iqnTarget + '\n').hexdigest()[0:8] logger.info("Looking for %s in lvDict %s" % (lvName, str(lvDict))) if lvName in lvDict: newLV = LV( target=newTarget, vg=chosenVG, lvname=lvName, lvsize=storageSize, #lvthinmapped=lvDict[lvName]['Mapped size'], lvuuid=lvDict[lvName]['LV UUID']) newLV.save() chosenVG.CurrentAllocGB = max( 0, chosenVG.CurrentAllocGB) + float(storageSize) chosenVG.maxavlGB = max( 0, chosenVG.maxavlGB - float(storageSize)) chosenVG.save() else: logger.error( 'Error - could not use ParseSCSTConf while working with target creation of %s, check if git and %s are in sync' % (iqnTarget, targethost + '.scst.conf')) return (-1, "CreateTarget returned error 2, contact admin") tar = Target.objects.get(iqntar=iqnTarget) aa = AAGroup(name=aagroup, target=tar) aa.save() aa.hosts.add(targethost) aa.save() newTarget.aagroup = aa cg = ClumpGroup(name=clumpgroup, target=tar) cg.save() cg.hosts.add(targethost) cg.save() newTarget.clumpgroup = cg newTarget.save() connection.close( ) #close DB connection to prevent RQ connection reset error in PG database logs return (0, iqnTarget) else: connection.close( ) #close DB connection to prevent RQ connection reset error in PG database logs logger.error('CreateTarget did not work') return (-1, "CreateTarget returned error 1, contact admin")
def directionSearch(request): start = request.POST['start'] goal = request.POST['goal'] print("Check Post -", start, goal) startGeo = getGeocode(start) goalGeo = getGeocode(goal) if startGeo == [] or goalGeo == [] : list = "NoData" return JsonResponse(list, safe=False) startLocation = startGeo[2] + "," + startGeo[3] goalLocation = goalGeo[2] + "," + goalGeo[3] print(startLocation) print(goalLocation) directionDataList = getDirectionApi(startLocation, goalLocation) print(directionDataList) cursor = connection.cursor() list = [] for directionData in directionDataList: latitude = str(directionData[1]) longtitude = str(directionData[0]) # print(latitude) # print(longtitude) try: strSql = "select evst.statNm,evst.addr,evst.lat,evst.lng,evst.useTime,evst.busicall,descInfo,congestion,(6371*acos(cos(radians(" + latitude + "))*cos(radians(evst.lat))*cos(radians(evst.lng)-radians(" + longtitude + "))+sin(radians(" + latitude + "))*sin(radians(evst.lat))))AS distance, info.descInfo from ev.ev_station evst join ev_real_time evtm on(evst.evsn=evtm.evsn),(select c.evSn, group_concat(des SEPARATOR '\n') as descInfo from (select a.evSn, a.chgerId, concat('기기 번호 :', a.chgerId , ' ( 상태 : ' , (select codeName from ev.ev_code_inf where codeId = a.stat) , ', 충전타입 : ', GROUP_CONCAT((select codeName from ev.ev_code_inf where codeId = b.chgerType) SEPARATOR ','),')') as des from ev.ev_station_status a, ev.ev_station_chgertype b where a.evSn = b.evSn group by a.evSn, a.chgerId) c group by c.evSn) info where evst.evSn = info.evSn HAVING distance <= 1 ORDER BY distance;" result = cursor.execute(strSql) stations = cursor.fetchall() print('stations - ', stations) cnt = 0 for station in stations: row = {'statnm': station[0], 'addr': station[1], 'lat': station[2], 'lng': station[3], 'useTime': station[4], 'busicall': station[5], 'info': station[6], 'congestion': station[7], } list.append(row) cnt += 1 if cnt == 3: break except: connection.rollback() print('Failed selecting in stations') connection.commit() connection.close() # for a in list: # print("check - ", a) # print(len(list)) return JsonResponse(list, safe=False)
def run_async_rich_menu_check(self): if not self.current_rich_menu: self.set_rich_menu(force_refresh=False) from django.db import connection connection.close()
def drop_db(): # close django's connection to the database connection.close() return subprocess.check_call(DROP_DB_CMD, shell=True)
def update_version_tuple(self): if get_database_version is None: if not hasattr(self, "_model_history"): # list of (model, database) versions self._model_history = [("0", "0")] self.model_version_mismatch = False # do not log, always true on client # self.log("cannot get model version, file missing ... ?", logging_tools.LOG_LEVEL_WARN) else: _database_v = get_database_version() _model_v = get_models_version() if not hasattr(self, "_model_history"): self._model_history = [(_model_v, _database_v)] self.model_version_mismatch = False self.log( "Starting Model version is {}, database version is {}". format( self._model_history[0][0], self._model_history[0][1], )) else: # this is only a rough start, we have to add grace periods and restart options, TODO, FIXME if _model_v != self._model_history[-1][0]: _cs = "Model version changed from {} to {} (database from version {} version is {}, history has now {:d} entries)".format( self._model_history[-1][0], _model_v, self._model_history[-1][1], _database_v, len(self._model_history) + 1, ) self._model_history.append((_model_v, _database_v)) self.model_version_mismatch = True if not is_debug_run(): self.log("{}".format(_cs), logging_tools.LOG_LEVEL_ERROR) else: self.log( "{} (ignoring due to debug run)".format(_cs, ), logging_tools.LOG_LEVEL_WARN) if self.model_version_mismatch and ICSWVersion is not None: try: # close database connection to disable query cache connection.close() except: pass _highest = self._model_history[-1] try: _h_idx = ICSWVersion.objects.all().order_by( "-insert_idx").values_list("insert_idx", flat=True)[0] except: self.log( "unable to determine highest ICSWVersion.insert_idx: {}" .format(process_tools.get_except_info()), logging_tools.LOG_LEVEL_ERROR) else: _dict = { _entry.name: _entry.version for _entry in ICSWVersion.objects.filter( Q(insert_idx=_h_idx)) } _db_model_v, _db_database_v = (_dict["models"], _dict["database"]) _v_info = "models={}, database={}".format( _db_model_v, _db_database_v, ) if _db_model_v != _highest[ 0] and _db_database_v != _highest[1]: self.log( "Version info from database ({}) does not match discovered info" .format(_v_info), logging_tools.LOG_LEVEL_WARN) else: self.log( "Version info from database ({}) matches discovered info, clearing mismatch flag" .format(_v_info)) self.model_version_mismatch = False
def query(request): cluster_name = request.POST.get('cluster_name') sqlContent = request.POST.get('sql_content') dbName = request.POST.get('db_name') limit_num = request.POST.get('limit_num') finalResult = {'status': 0, 'msg': 'ok', 'data': {}} # 服务器端参数验证 if sqlContent is None or dbName is None or cluster_name is None or limit_num is None: finalResult['status'] = 1 finalResult['msg'] = '页面提交参数可能为空' return HttpResponse(json.dumps(finalResult), content_type='application/json') sqlContent = sqlContent.strip() if sqlContent[-1] != ";": finalResult['status'] = 1 finalResult['msg'] = 'SQL语句结尾没有以;结尾,请重新修改并提交!' return HttpResponse(json.dumps(finalResult), content_type='application/json') # 获取用户信息 loginUser = request.session.get('login_username', False) loginUserOb = users.objects.get(username=loginUser) # 过滤注释语句和非查询的语句 sqlContent = ''.join( map(lambda x: re.compile(r'(^--\s+.*|^/\*.*\*/;\s*$)').sub('', x, count=1), sqlContent.splitlines(1))).strip() # 去除空行 sqlContent = re.sub('[\r\n\f]{2,}', '\n', sqlContent) sql_list = sqlContent.strip().split('\n') for sql in sql_list: if re.match(r"^select|^show|^explain", sql.lower()): break else: finalResult['status'] = 1 finalResult['msg'] = '仅支持^select|^show|^explain语法,请联系管理员!' return HttpResponse(json.dumps(finalResult), content_type='application/json') # 取出该集群的连接方式,查询只读账号,按照分号截取第一条有效sql执行 slave_info = slave_config.objects.get(cluster_name=cluster_name) sqlContent = sqlContent.strip().split(';')[0] # 查询权限校验,以及limit_num获取 priv_check_info = query_priv_check(loginUserOb, cluster_name, dbName, sqlContent, limit_num) if priv_check_info['status'] == 0: limit_num = priv_check_info['data'] else: return HttpResponse(json.dumps(priv_check_info), content_type='application/json') if re.match(r"^explain", sqlContent.lower()): limit_num = 0 # 对查询sql增加limit限制 if re.match(r"^select", sqlContent.lower()): if re.search(r"limit\s+(\d+)$", sqlContent.lower()) is None: if re.search(r"limit\s+\d+\s*,\s*(\d+)$", sqlContent.lower()) is None: sqlContent = sqlContent + ' limit ' + str(limit_num) sqlContent = sqlContent + ';' # 执行查询语句,统计执行时间 t_start = time.time() sql_result = dao.mysql_query(slave_info.slave_host, slave_info.slave_port, slave_info.slave_user, prpCryptor.decrypt(slave_info.slave_password), str(dbName), sqlContent, limit_num) t_end = time.time() cost_time = "%5s" % "{:.4f}".format(t_end - t_start) sql_result['cost_time'] = cost_time # 数据脱敏,同样需要检查配置,是否开启脱敏,语法树解析是否允许出错继续执行 t_start = time.time() if settings.DATA_MASKING_ON_OFF: # 仅对查询语句进行脱敏 if re.match(r"^select", sqlContent.lower()): try: masking_result = datamasking.data_masking(cluster_name, dbName, sqlContent, sql_result) except Exception: if settings.CHECK_QUERY_ON_OFF: finalResult['status'] = 1 finalResult['msg'] = '脱敏数据报错,请联系管理员' return HttpResponse(json.dumps(finalResult), content_type='application/json') else: if masking_result['status'] != 0: if settings.CHECK_QUERY_ON_OFF: return HttpResponse(json.dumps(masking_result), content_type='application/json') t_end = time.time() masking_cost_time = "%5s" % "{:.4f}".format(t_end - t_start) sql_result['masking_cost_time'] = masking_cost_time finalResult['data'] = sql_result # 成功的查询语句记录存入数据库 if sql_result.get('Error'): pass else: query_log = QueryLog() query_log.username = loginUser query_log.db_name = dbName query_log.cluster_name = cluster_name query_log.sqllog = sqlContent if int(limit_num) == 0: limit_num = int(sql_result['effect_row']) else: limit_num = min(int(limit_num), int(sql_result['effect_row'])) query_log.effect_row = limit_num query_log.cost_time = cost_time # 防止查询超时 try: query_log.save() except: connection.close() query_log.save() # 返回查询结果 return HttpResponse(json.dumps(finalResult, cls=ExtendJSONEncoder, bigint_as_string=True), content_type='application/json')
def run_cron(jobs=None, update_heartbeat=True, force_run=False, dryrun=False, clear_pid=False): try: # TODO: auto-kill inactive long-running cron processes whose # threads have stalled and not exited properly? # Check for 0 cpu usage. #ps -p <pid> -o %cpu stdout_map = defaultdict(list) # {prod_id:[]} stderr_map = defaultdict(list) # {prod_id:[]} stdout_queue = Queue() stderr_queue = Queue() if settings.CHRONIKER_AUTO_END_STALE_JOBS and not dryrun: Job.objects.end_all_stale() # Check PID file to prevent conflicts with prior executions. # TODO: is this still necessary? deprecate? As long as jobs run by # JobProcess don't wait for other jobs, multiple instances of cron # should be able to run simeltaneously without issue. if settings.CHRONIKER_USE_PID: pid_fn = settings.CHRONIKER_PID_FN pid = str(os.getpid()) any_running = Job.objects.all_running().count() if not any_running: # If no jobs are running, then even if the PID file exists, # it must be stale, so ignore it. pass elif os.path.isfile(pid_fn): try: old_pid = int(open(pid_fn, 'r').read()) if utils.pid_exists(old_pid): print('%s already exists, exiting' % pid_fn) sys.exit() else: print(('%s already exists, but contains stale ' 'PID, continuing') % pid_fn) except ValueError: pass except TypeError: pass file(pid_fn, 'w').write(pid) clear_pid = True procs = [] if force_run: q = Job.objects.all() if jobs: q = q.filter(id__in=jobs) else: q = Job.objects.due_with_met_dependencies_ordered(jobs=jobs) running_ids = set() for job in q: # This is necessary, otherwise we get the exception # DatabaseError: SSL error: sslv3 alert bad record mac # even through we're not using SSL... # We work around this by forcing Django to use separate # connections for each process by explicitly closing the # current connection. connection.close() # Re-check dependencies to incorporate any previous iterations # that marked jobs as running, potentially causing dependencies # to become unmet. Job.objects.update() job = Job.objects.get(id=job.id) if not force_run and not job.is_due_with_dependencies_met( running_ids=running_ids): print('Job %i %s is due but has unmet dependencies.' % (job.id, job)) continue # Immediately mark the job as running so the next jobs can # update their dependency check. print('Running job %i %s.' % (job.id, job)) running_ids.add(job.id) if dryrun: continue job.is_running = True Job.objects.filter(id=job.id).update(is_running=job.is_running) # Launch job. #proc = JobProcess(job, update_heartbeat=update_heartbeat, name=str(job)) job_func = partial( run_job, job=job, force_run=force_run or job.force_run, update_heartbeat=update_heartbeat, name=str(job), ) proc = JobProcess(job=job, max_seconds=job.timeout_seconds, target=job_func, name=str(job), kwargs=dict( stdout_queue=stdout_queue, stderr_queue=stderr_queue, )) proc.start() procs.append(proc) if not dryrun: print("%d Jobs are due." % len(procs)) # Wait for all job processes to complete. while procs: while not stdout_queue.empty(): proc_id, proc_stdout = stdout_queue.get() stdout_map[proc_id].append(proc_stdout) while not stderr_queue.empty(): proc_id, proc_stderr = stderr_queue.get() stderr_map[proc_id].append(proc_stderr) for proc in list(procs): # Auto kill processes that haven't terminated but yet # register no cpu usage. #cpu = proc.get_cpu_usage_recursive() #print('cpu:',proc,cpu) # if not cpu: # utils.kill_process(proc.pid) # time.sleep(1) if not proc.is_alive(): print('Process %s ended.' % (proc, )) procs.remove(proc) elif proc.is_expired: print('Process %s expired.' % (proc, )) proc_id = proc.pid proc.terminate() run_end_datetime = timezone.now() procs.remove(proc) connection.close() Job.objects.update() run_start_datetime = Job.objects.get( id=proc.job.id).last_run_start_timestamp proc.job.is_running = False proc.job.force_run = False proc.job.force_stop = False proc.job.save() # Create log record since the job was killed before it had # a chance to do so. Log.objects.create( job=proc.job, run_start_datetime=run_start_datetime, run_end_datetime=run_end_datetime, success=False, on_time=False, hostname=socket.gethostname(), stdout=''.join(stdout_map[proc_id]), stderr=''.join(stderr_map[proc_id] + ['Job exceeded timeout\n']), ) time.sleep(1) print('!' * 80) print('All jobs complete!') finally: if settings.CHRONIKER_USE_PID and os.path.isfile(pid_fn) \ and clear_pid: os.unlink(pid_fn)
def _decorator(*args, **kwargs): try: return function(*args, **kwargs) finally: connection.close()
def execute(self, workflow=None): """执行上线单""" instance = workflow.instance execute_result = ReviewSet( full_sql=workflow.sqlworkflowcontent.sql_content) inception_engine = InceptionEngine() if workflow.is_backup: str_backup = "--enable-remote-backup" else: str_backup = "--disable-remote-backup" # 根据inception的要求,执行之前最好先split一下 sql_split = f"""/*--user={instance.user};--password={instance.raw_password};--host={instance.host}; --port={instance.port};--enable-ignore-warnings;--enable-split;*/ inception_magic_start; use {workflow.db_name}; {workflow.sqlworkflowcontent.sql_content} inception_magic_commit;""" split_result = inception_engine.query(sql=sql_split) execute_result.rows = [] # 对于split好的结果,再次交给inception执行,保持长连接里执行. for splitRow in split_result.rows: sql_tmp = splitRow[1] sql_execute = f"""/*--user={instance.user};--password={instance.raw_password};--host={instance.host}; --port={instance.port};--enable-execute;--enable-ignore-warnings;{str_backup};*/\ inception_magic_start;\ {sql_tmp}\ inception_magic_commit;""" one_line_execute_result = inception_engine.query(sql=sql_execute, close_conn=False) # 执行, 把结果转换为ReviewSet for sqlRow in one_line_execute_result.to_dict(): execute_result.rows.append( ReviewResult(id=sqlRow['ID'], stage=sqlRow['stage'], errlevel=sqlRow['errlevel'], stagestatus=sqlRow['stagestatus'], errormessage=sqlRow['errormessage'], sql=sqlRow['SQL'], affected_rows=sqlRow['Affected_rows'], actual_affected_rows=sqlRow['Affected_rows'], sequence=sqlRow['sequence'], backup_dbname=sqlRow['backup_dbname'], execute_time=sqlRow['execute_time'], sqlsha1=sqlRow['sqlsha1'])) # 每执行一次,就将执行结果更新到工单的execute_result,便于展示执行进度和保存执行信息 workflow.sqlworkflowcontent.execute_result = execute_result.json() try: workflow.sqlworkflowcontent.save() workflow.save() # 防止执行超时 except OperationalError: connection.close() workflow.sqlworkflowcontent.save() workflow.save() # 如果发现任何一个行执行结果里有errLevel为1或2,并且stagestatus列没有包含Execute Successfully字样,则最终执行结果为有异常. execute_result.status = "workflow_finish" for sqlRow in execute_result.rows: if sqlRow.errlevel in (1, 2) and not re.search( r"Execute Successfully", sqlRow.stagestatus): execute_result.status = "workflow_exception" execute_result.error = "Line {0} has error/warning: {1}".format( sqlRow.id, sqlRow.errormessage) break return execute_result
def generate_tags(article_id): connection.close() try: import pandas as pd import numpy as np import itertools from sklearn.feature_extraction.text import TfidfTransformer, CountVectorizer from sklearn.preprocessing import MultiLabelBinarizer from sklearn.multiclass import OneVsRestClassifier from sklearn.svm import SVC from website.models import Article a = Article.objects.get(id=article_id) # import article's comments into dataframe df = pd.DataFrame( list(a.comment_set.all().values('id', 'article', 'disqus_id', 'text', 'summary', 'tags', 'suggested_tags'))) # merge all text (comments+summaries) into a new column df['train_text'] = df[['text', 'summary']].apply(lambda x: ' '.join(x), axis=1) # define classifier clf = OneVsRestClassifier(SVC(kernel='linear')) # train data: use only comments with tags tagged = df.loc[df['tags'].notnull()] # train data: preproccess and vectorize (TfIdf) text data count_vect = CountVectorizer( stop_words='english', min_df=3, max_df=0.30, #lowercase=True, ngram_range=(1, 2), ) X_train_counts = count_vect.fit_transform(list(tagged.train_text)) tfidf_transformer = TfidfTransformer().fit(X_train_counts) X_train_tfidf = tfidf_transformer.transform(X_train_counts) # train classifier clf = clf.fit(X_train_tfidf, tagged.tags) # suggest tags for ALL instances in df test_df = df.drop_duplicates(subset=['disqus_id']) X_test_counts = count_vect.transform(list(test_df.train_text)) X_test_tfidf = tfidf_transformer.transform(X_test_counts) suggested = clf.predict(X_test_tfidf) # save suggested tags to the dataframe test_df.suggested_tags = suggested # add suggested tags to the database sorted_df = test_df.sort_values('disqus_id') comments = a.comment_set.all().order_by('disqus_id') for comment in comments: comment.suggested_tags.clear() for row_item, comment in zip(sorted_df.iterrows(), comments): index, row = row_item if row['suggested_tags']: comment.suggested_tags.add(row['suggested_tags']) except Exception, e: print e
def ThreadMain(self, ProcName): logger.info('Starting ' + ProcName) # Start Process pool with 2 process self.ReqTags = 2 self.ProcPool = multiprocessing.Pool(self.ReqTags) jobs = [] while 1: if self.mDieFlag == 1: break # Request for death self.mLock.acquire() self.ProcPoolProblemFlag = 0 for self.worker in self.ProcPool._pool: if not self.worker.is_alive(): self.ProcPoolProblemFlag = 1 logger.error( 'Problem with process_name: %s, process_pid: %s, process_exitcode: %s', self.worker.name, self.worker.pid, self.worker.exitcode) self.Time, self.Run = ESSDB.DB().action('ESSProc', 'GET', ('Time', 'Run'), ('Name', ProcName))[0] if self.Run == '0' or self.ProcPoolProblemFlag == 1: logger.info('Stopping ' + ProcName) if self.ProcPoolProblemFlag: self.ProcPool.terminate() else: self.ProcPool.close() self.ProcPool.join() ESSDB.DB().action('ESSProc', 'UPD', ('Status', '0', 'Run', '0', 'PID', '0'), ('Name', ProcName)) time.sleep(1) self.mLock.release() logger.info('RunFlag: 0') time.sleep(1) break # Process Item lock = thread.allocate_lock() AccessQueue_DbRows = AccessQueue.objects.filter(Status=0).all() for AccessQueue_DbRow in AccessQueue_DbRows: ##############################################################################################Y # if self.ProcPool._state == 0 then pool is working. if self.ProcPool._state == 0: # Get active queue depth for self.ProcPool._cache. self.ActiveProcQueue = len(self.ProcPool._cache) ########################################################################################### # If self.ActiveProcQueue < self.ReqTags start DIPRequest process if self.ActiveProcQueue < self.ReqTags: AccessQueue_DbRow.Status = 2 #model.meta.Session.commit() AccessQueue_DbRow.save() logger.info('Add ReqUUID: %s to GenerateDIPProc' % AccessQueue_DbRow.ReqUUID) res = self.ProcPool.apply_async( GenerateDIPProc, (AccessQueue_DbRow.ReqUUID, )) jobs.append(res) for job in jobs: try: msg = 'Result from GenerateDIPProc: %s' % repr( job.get(timeout=1)) except multiprocessing.TimeoutError as e: msg = 'Timeout wait for result from GenerateDIPProc' logger.debug(msg) if len(self.ProcPool._cache) == 0: jobs = [] logger.debug('ProcPool_cache: %r', self.ProcPool._cache) connection.close() time.sleep(5) self.mLock.release() time.sleep(10) self.RunFlag = 0 self.mDieFlag = 0
except Exception, ee: print '--ee:', ee else: q = queqe_server() cursor = connection.cursor() print '-----------------------Start Writedata %s' % index if AutoDelTmp: try: ret = auto_del() except Exception, e: print '--Auto Check&Delete failed!(%s):' % e parseLogDataInFile(connection, cursor) try: cursor.close() connection.close() except: pass q.connection.disconnect() print '-----------------------End Writedata %s' % index class WriteDataThread(threading.Thread): def __init__(self, index): self.index = index super(WriteDataThread, self).__init__() def run(self): run_writedata(self.index)
def info_list(request): query = request.GET['query'] champ_df = pd.read_csv('D:/KUGG/kugg/df/Champ.csv') champ_list=list(champ_df['name'].values) if query: #챔피언 검색 if query in champ_list: #cid=122 for i,r in enumerate(champ_df['name']): if query == r: cid = champ_df['champion'][i] item_tree = get_item_tree(int(cid)) #print(item_tree) context = {'selected_champ' : query, 'champ_list':champ_list, 'item_tree':item_tree} print("search champion") return render(request, 'kugg/selected_champion_list.html', context) #소환사명 검색 try: curs = conn.cursor() sql = "select * from usersinfo where sname=(%s)" curs.execute(sql,(query,)) usersinfo = curs.fetchone() summonerId = usersinfo[0] accountId = usersinfo[1] sname = usersinfo[3] profileIconId = usersinfo[4] revisionDate_unix = usersinfo[5]/1000 revisionDate = convert_datetime(revisionDate_unix) summonerLevel = usersinfo[6] print("userinfo complete") except: conn.rollback() print("Failed selecting in usersinfo") context = {"username": query} return render(request, 'kugg/info_list.html',context) try: sql = "select * from usersleague where summonerId=(%s)" curs.execute(sql,(summonerId,)) usersleague = curs.fetchone() tier = usersleague[0] #leagueId = usersleague[1] queue = usersleague[2] #게임종류 lname = usersleague[3] #리그이름 leaguePoints = usersleague[6] srank = usersleague[7] #마스터 1에서 숫자부분 wins = usersleague[8] lossers = usersleague[9] veteran = usersleague[10] win_rate = math.trunc(wins*100/(wins+lossers)) info_df = pd.DataFrame(columns=['sname','profileIconId','revisionDate','summonerLevel','tier','queue','lname','leaguePoints','srank','wins','lossers','veteran','win_rate']) info_df.loc[0]=[sname,profileIconId,revisionDate,summonerLevel,tier,queue,lname,leaguePoints,srank,wins,lossers,veteran,win_rate] #print(info_df) # info_json_records = info_df.reset_index().to_json(orient ='records') # info_data = [] # info_data = json.loads(info_json_records) info_data = df_to_json(info_df) #print(info_data) print("uesrleague complete") except: conn.rollback() info_df = pd.DataFrame(columns=['sname','profileIconId','revisionDate','summonerLevel']) info_df.loc[0]=[sname,profileIconId,revisionDate,summonerLevel] #print(info_df) # info_json_records = info_df.reset_index().to_json(orient ='records') # info_data = [] # info_data = json.loads(info_json_records) info_data = df_to_json(info_df) print("Failed selecting in usersleague") try: sql = "select * from usersmatchlist_sample where accountId=(%s) limit 20" curs.execute(sql,(accountId,)) data = curs.fetchall() platformId=[] gameId=[] champion_tmp=[] queue=[] season=[] timestamp=[] role=[] lane=[] for t in data: platformId.append(t[0]) gameId.append(t[1]) champion_tmp.append(t[2]) queue.append(t[3]) season.append(t[4]) timestamp.append(t[5]) role.append(t[6]) lane.append(t[7]) champion=[] for c in champion_tmp: for i,r in enumerate(champ_df['champion']): if c == r: champion.append(champ_df['name'][i]) #print(champion) columns_matchlist = ['platformId','gameId','champion','queue','season','timestamp','role','lane'] matchlists = [] gameId_len = len(gameId) for idx in range(gameId_len): matchlist = [platformId[idx],gameId[idx],champion[idx],queue[idx],season[idx],timestamp[idx],role[idx],lane[idx]] matchlists.append(matchlist) print("matchlist complete") #print(matchlists) except: conn.rollback() print("Failed selecting in usersmatchlist") try: #gameId = gameId[-1::-1] matchlists = matchlists[-1::-1] gameId = tuple(gameId) # gameId = (4684958376, 4684716231, 4681708889) # gameId_len = 3 if gameId_len == 1: sql = "select * from matchinfo where gameMode='CLASSIC' and gameId=(%s)" #게임이 여러개면 or연산 elif gameId_len ==2: sql = "select * from matchinfo where gameMode='CLASSIC' and gameId=(%s) or gameId=(%s)" elif gameId_len ==3: sql = "select * from matchinfo where gameMode='CLASSIC' and gameId=(%s) or gameId=(%s) or gameId=(%s)" elif gameId_len ==4: sql = "select * from matchinfo where gameMode='CLASSIC' and (gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s))" elif gameId_len ==5: sql = "select * from matchinfo where gameMode='CLASSIC' and (gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s))" elif gameId_len ==6: sql = "select * from matchinfo where gameMode='CLASSIC' and (gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s))" elif gameId_len ==7: sql = "select * from matchinfo where gameMode='CLASSIC' and (gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s))" elif gameId_len ==8: sql = "select * from matchinfo where gameMode='CLASSIC' and (gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s))" elif gameId_len ==9: sql = "select * from matchinfo where gameMode='CLASSIC' and (gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s))" elif gameId_len ==10: sql = "select * from matchinfo where gameMode='CLASSIC' and (gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s))" elif gameId_len ==11: sql = "select * from matchinfo where gameMode='CLASSIC' and (gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s))" elif gameId_len ==12: sql = "select * from matchinfo where gameMode='CLASSIC' and (gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s))" elif gameId_len ==13: sql = "select * from matchinfo where gameMode='CLASSIC' and (gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s))" elif gameId_len ==14: sql = "select * from matchinfo where gameMode='CLASSIC' and (gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s))" elif gameId_len ==15: sql = "select * from matchinfo where gameMode='CLASSIC' and (gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s))" elif gameId_len ==16: sql = "select * from matchinfo where gameMode='CLASSIC' and (gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s))" elif gameId_len ==17: sql = "select * from matchinfo where gameMode='CLASSIC' and (gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s))" elif gameId_len ==18: sql = "select * from matchinfo where gameMode='CLASSIC' and (gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s))" elif gameId_len ==19: sql = "select * from matchinfo where gameMode='CLASSIC' and (gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s))" elif gameId_len ==20: print('gamId 20') sql = "select * from matchinfo where gameMode='CLASSIC' and (gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s) or gameId=(%s))" curs.execute(sql,gameId) print("sql complete") data_list=[] match_num=0 matchinfo = curs.fetchone() print("fetch complete") c = 0 comp_idx = 0 #print(matchlists[0]) gameId= gameId[-1::-1] #print(gameId) matchlists_comp=[] while matchinfo: #print(matchinfo[8]) for i,g in enumerate(gameId): if g == matchinfo[0]: comp_idx = i #print(gameId[comp_idx]) if matchinfo[8] == "CLASSIC" : #print("if CLASSIC") matchlists_comp.append(matchlists[comp_idx]) #print(matchinfo[0]) #print(matchinfo[8]) teams = matchinfo[10] participants = matchinfo[11] pId = matchinfo[12] print('teams,participants') # teams = teams.replace('\\','') # teams = teams[1:-1] # teams = literal_eval(teams) teams = json_to_df(teams) # participants = participants.replace('\\','') # participants = participants[1:-1] # participants = literal_eval(participants) participants = json_to_df(participants) # pId = pId.replace('\\','') # pId = pId[1:-1] # pId = literal_eval(pId) pId = json_to_df(pId) parts = [ pId[i]['player']['accountId'] for i in range(10)] #print(all_part) for i,a in enumerate(parts): if a == accountId: idx = i win_lose = participants[idx]['stats']['win'] if win_lose == True: game_win = "승리" elif win_lose == False: game_win = "패배" spell1Id = participants[idx]['spell1Id'] spell2Id = participants[idx]['spell2Id'] perkPrimaryStyle = participants[idx]['stats']['perkPrimaryStyle'] perkSubStyle = participants[idx]['stats']['perkSubStyle'] item0 = participants[idx]['stats']['item0'] item1 = participants[idx]['stats']['item1'] item2 = participants[idx]['stats']['item2'] item3 = participants[idx]['stats']['item3'] item4 = participants[idx]['stats']['item4'] item5 = participants[idx]['stats']['item5'] item6 = participants[idx]['stats']['item6'] item = [item0,item1,item2,item3,item4,item5,item6] visionWards = participants[idx]['stats']['visionWardsBoughtInGame'] visionScore = participants[idx]['stats']['visionScore'] kills = participants[idx]['stats']['kills'] deaths = participants[idx]['stats']['deaths'] assists = participants[idx]['stats']['assists'] kda = round((kills + assists)/deaths,2) if deaths != 0 else 'perfect' penta = participants[idx]['stats']['pentaKills'] quadra = participants[idx]['stats']['quadraKills'] triple = participants[idx]['stats']['tripleKills'] double = participants[idx]['stats']['doubleKills'] if penta >= 1: killmsg = '펜타킬' elif quadra >=1: killmsg = '쿼드라킬' elif triple >=1: killmsg = '트리플킬' elif double >=1: killmsg = '더블킬' else: killmsg = 0 totalDamageDealt = participants[idx]['stats']['totalDamageDealt'] totalMinionsKilled = participants[idx]['stats']['totalMinionsKilled'] champLevel = participants[idx]['stats']['champLevel'] # pname=[] # for p in parts: # sql = "select * from usersinfo where accountId=(%s)" # curs.execute(sql,(p,)) # data = curs.fetchone() # pname.append(data[7]) # print(pname) p1champ = participants[0]['championId'] p2champ = participants[1]['championId'] p3champ = participants[2]['championId'] p4champ = participants[3]['championId'] p5champ = participants[4]['championId'] p6champ = participants[5]['championId'] p7champ = participants[6]['championId'] p8champ = participants[7]['championId'] p9champ = participants[8]['championId'] p10champ = participants[9]['championId'] pchamp_tmp = [p1champ,p2champ,p3champ,p4champ,p5champ,p6champ,p7champ,p8champ,p9champ,p10champ] #print(p1champ) gameCreation = matchinfo[2] gameDurationH = math.trunc(matchinfo[3]/60) gameDurationS = matchinfo[3]%60 queueId = matchinfo[4] mapId = matchinfo[5] seasonId = matchinfo[6] gameVersion = matchinfo[7][:5] gameMode = matchinfo[8] gameType = matchinfo[9] info = [gameCreation,gameDurationH,gameDurationS,queueId,mapId,seasonId,gameVersion,gameMode,gameType] #print(info) #print(teams[0]['bans']) bans_tmp = [teams[i]['bans'][j]['championId'] for i in range(2) for j in range(5)] #print(bans_tmp) pchamp=[] bans=[] for c,b in zip(pchamp_tmp,bans_tmp): for i,r in enumerate(champ_df['champion']): if c == r: pchamp.append(champ_df['name'][i]) if b == r: bans.append(champ_df['name'][i]) #print(bans) part = [spell1Id,spell2Id,perkPrimaryStyle,perkSubStyle,item,visionWards,visionScore,kills,deaths,assists,kda,killmsg,totalDamageDealt,totalMinionsKilled,champLevel,pchamp,bans,game_win] #,lane] match = matchlists_comp[match_num] + info + part #+ bans c+=1 match_num += 1 columns_info = ['gameCreation','gameDurationH','gameDurationS','queueId','mapId','seasonId','gameVersion','gameMode','gameType'] #columns_bans = ['ban1','ban2','ban3','ban4','ban5','ban6','ban7','ban8','ban9','ban10'] columns_part = ['spell1Id','spell2Id','perkPrimaryStyle','perkSubStyle','item','visionWardsBoughtInGame','visionScore','kills','deaths','assists','kda','killmsg','totalDamageDealt','totalMinionsKilled','champLevel','pchamp','bans','game_win'] #,'lane'] columns = columns_matchlist + columns_info + columns_part #+ columns_bans print("dataframe") #print(game_win) df = pd.DataFrame(columns=columns) df.loc[0]=match #df에 한줄씩 추가 # json_records = df.reset_index().to_json(orient ='records') # data = [] # data = json.loads(json_records) data = df_to_json(df) #json들의 list를 만들어서 20개 json을 담는다 data_list.append(data) matchinfo = curs.fetchone() print("all complete") #print(data_list) #print(info_data) context = {'match_list': data_list, 'info' : info_data} curs.close() conn.close() except: conn.rollback() print("Failed selecting in matchinfo") context = {'context' : 'NULL'} else: err = "소환사명 또는 챔피언명을 입력해주세요" context = {'err' : err} return render(request, 'kugg/info_list.html', context)
def ProcessAccessRequest(self, ReqUUID): """Process access request :param ReqUUID: ReqUUID in database table AccessQueue """ try: logger.debug('Start ProcessAccessRequest') logger.debug('ReqUUID: %s' % ReqUUID) connection.close() # Fix (2006, 'MySQL server has gone away') AccessQueue_obj = AccessQueue.objects.get(ReqUUID=ReqUUID) process_name = multiprocessing.current_process().name logger.debug('process_name: %s' % process_name) process_pid = multiprocessing.current_process().pid logger.debug('process_pid: %s' % process_pid) AccessQueue_obj.Status = 5 AccessQueue_obj.save() if AccessQueue_obj.ReqType in (1, 3, 4, 5): event_info = 'Start Generate DIP Process for ObjectIdentifierValue: %s, ReqUUID: %s' % ( AccessQueue_obj.ObjectIdentifierValue, AccessQueue_obj.ReqUUID) logger.info(event_info) ESSPGM.Events().create('1202', AccessQueue_obj.ReqPurpose, 'ESSArch Access', ProcVersion, '0', event_info, 2, AccessQueue_obj.ObjectIdentifierValue) elif AccessQueue_obj.ReqType == 2: event_info = 'Start quickverify storageMediumID Process for storageMediumID: %s, ReqUUID: %s' % ( AccessQueue_obj.storageMediumID, AccessQueue_obj.ReqUUID) logger.info(event_info) ESSPGM.Events().create('2202', AccessQueue_obj.ReqPurpose, 'ESSArch Access', ProcVersion, '0', event_info, 2, AccessQueue_obj.ObjectIdentifierValue) StorageMethodRead_obj = StorageMethodRead() StorageMethodRead_obj.logger = logger StorageMethodRead_obj.AccessQueue_obj = AccessQueue_obj if AccessQueue_obj.ReqType == 1: StorageMethodRead_obj.get_object_to_read() StorageMethodRead_obj.add_to_ioqueue() StorageMethodRead_obj.apply_ios_to_read() StorageMethodRead_obj.wait_for_all_reads() StorageMethodRead_obj.ip_unpack() StorageMethodRead_obj.ip_validate() elif AccessQueue_obj.ReqType == 3: StorageMethodRead_obj.get_object_to_read() StorageMethodRead_obj.add_to_ioqueue() StorageMethodRead_obj.apply_ios_to_read() StorageMethodRead_obj.wait_for_all_reads() elif AccessQueue_obj.ReqType in [4, 5]: StorageMethodRead_obj.get_object_to_read() StorageMethodRead_obj.add_to_ioqueue() StorageMethodRead_obj.apply_ios_to_read() StorageMethodRead_obj.wait_for_all_reads() StorageMethodRead_obj.ip_unpack() StorageMethodRead_obj.ip_validate() StorageMethodRead_obj.delete_retrieved_ios() elif AccessQueue_obj.ReqType == 2: StorageMethodRead_obj.get_objects_to_verify() StorageMethodRead_obj.add_to_ioqueue() StorageMethodRead_obj.apply_ios_to_read() StorageMethodRead_obj.wait_for_all_reads() StorageMethodRead_obj.delete_retrieved_ios() except AccessError as e: exc_type, exc_value, exc_traceback = sys.exc_info() AccessQueue_obj.refresh_from_db() if AccessQueue_obj.ReqType in (1, 3, 4, 5): event_info = 'Problem to Generate DIP for ObjectIdentifierValue: %s, ReqUUID: %s, error: %s, line: %s' % ( AccessQueue_obj.ObjectIdentifierValue, AccessQueue_obj.ReqUUID, e, exc_traceback.tb_lineno) logger.error(event_info) ESSPGM.Events().create('1203', AccessQueue_obj.ReqPurpose, 'ESSArch Access', ProcVersion, '1', event_info, 2, AccessQueue_obj.ObjectIdentifierValue) elif AccessQueue_obj.ReqType == 2: event_info = 'Problem to quickverify storageMediumID: %s, ReqUUID: %s, error: %s line: %s' % ( AccessQueue_obj.storageMediumID, AccessQueue_obj.ReqUUID, e, exc_traceback.tb_lineno) logger.error(event_info) ESSPGM.Events().create( '2203', AccessQueue_obj.ReqPurpose, 'ESSArch Access', ProcVersion, '1', event_info, 2, storageMediumID=AccessQueue_obj.storageMediumID) AccessQueue_obj.Status = 100 AccessQueue_obj.save(update_fields=['Status']) #raise e except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() msg = 'Unknown error, error: %s trace: %s' % ( e, repr(traceback.format_tb(exc_traceback))) logger.error(msg) AccessQueue_obj.refresh_from_db() msg = 'Unknown error with access ReqUUID: %s, error: %s trace: %s' % ( AccessQueue_obj.ReqUUID, e, repr(traceback.format_tb(exc_traceback))) logger.error(msg) AccessQueue_obj.Status = 100 AccessQueue_obj.save(update_fields=['Status']) #raise e except: msg = 'Unexpected error: %s %s' % (sys.exc_info()[0], sys.exc_info()[1]) logger.error(msg) print msg #raise else: if AccessQueue_obj.ReqType in (1, 3, 4): event_info = 'Success to Generate DIP for ObjectIdentifierValue: %s, ReqUUID: %s' % ( AccessQueue_obj.ObjectIdentifierValue, AccessQueue_obj.ReqUUID) logger.info(event_info) ESSPGM.Events().create('1203', AccessQueue_obj.ReqPurpose, 'ESSArch Access', ProcVersion, '0', event_info, 2, AccessQueue_obj.ObjectIdentifierValue) elif AccessQueue_obj.ReqType == 5: event_info = 'Success to get AIP to ControlArea for ObjectIdentifierValue: %s, ReqUUID: %s' % ( AccessQueue_obj.ObjectIdentifierValue, AccessQueue_obj.ReqUUID) logger.info(event_info) ESSPGM.Events().create('1203', AccessQueue_obj.ReqPurpose, 'ESSArch Access', ProcVersion, '0', event_info, 2, AccessQueue_obj.ObjectIdentifierValue) # Update IP in ArchiveObject DBtable ArchiveObject_upd = ArchiveObject.objects.get( ObjectIdentifierValue=AccessQueue_obj.ObjectIdentifierValue ) setattr(ArchiveObject_upd, 'StatusActivity', 7) # Commit DB updates ArchiveObject_upd.save(update_fields=['StatusActivity']) elif AccessQueue_obj.ReqType == 2: event_info = 'Success to quickverify storageMediumID: %s, ReqUUID: %s' % ( AccessQueue_obj.storageMediumID, AccessQueue_obj.ReqUUID) logger.info(event_info) ESSPGM.Events().create( '2203', AccessQueue_obj.ReqPurpose, 'ESSArch Access', ProcVersion, '0', event_info, 2, storageMediumID=AccessQueue_obj.storageMediumID) AccessQueue_obj.Status = 20 AccessQueue_obj.save(update_fields=['Status'])
def importActivitiesFromCSV(url): file_name = url.split('/')[-1] u = urllib2.urlopen(url) f = open(file_name, 'wb') meta = u.info() file_size = int(meta.getheaders("Content-Length")[0]) print "Downloading: %s Bytes: %s" % (file_name, file_size) file_size_dl = 0 block_sz = 8192 while True: buffer = u.read(block_sz) if not buffer: break file_size_dl += len(buffer) f.write(buffer) status = r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size) status = status + chr(8) * (len(status) + 1) print status, f.close() f = open(file_name, 'rb') csvDict = csv.DictReader(f) print csvDict print f print csvDict.fieldnames print 'File opened.' csvDict.fieldnames = [ field.strip().lower() for field in csvDict.fieldnames ] print 'Field names converted: %s' % str(csvDict.fieldnames) count = 0 existingCount = 0 errorCount = 0 print 'Starting line iteration.' for line in csvDict: # unicode_line = dict([(k.encode('utf8'), v.encode('utf8')) for k, v in line.items()]) newTime = oracleTimeToDateTime(line['start_time_local']) line['start_time_local'] = newTime newActivity = NikeSportActivity(**line) # objects_to_bulk_create.append(newActivity) # if len(objects_to_bulk_create) >= 10: # try: # NikeSportActivity.objects.bulk_create(objects_to_bulk_create) # count += len(objects_to_bulk_create) # objects_to_bulk_create = [] # except IntegrityError, e: # existingCount += len(objects_to_bulk_create) # objects_to_bulk_create = [] # print 'IntegrityError during bulk save: %s' % str(e) # except DatabaseError, e: # errorCount += len(objects_to_bulk_create) # objects_to_bulk_create = [] # print 'DatabaseError during bulk save: %s' % str(e) # print '%d rows completed. %d existing rows skipped. Error count: %d' % (count, existingCount, errorCount) try: newActivity.save() count += 1 except IntegrityError, e: # print e connection.close() existingCount += 1 except DatabaseError, e: # print e connection.close() errorCount += 1
def close_connection(**kwargs): connection.close()
def maybeCloseDatabaseConnections(self): """Close database connections if their use is not permitted.""" if self.database_use_possible and not self.database_use_permitted: from django.db import connection connection.close()
def generator(cursor): for row in cursor: yield Decimal(row[0]) cursor.close() connection.close()
def disconnect(self, close_code): # Leave room group if self.room_group_name: connection.close() async_to_sync(self.channel_layer.group_discard)( self.room_group_name, self.channel_name)
def importUsersFromCSV(url): file_name = url.split('/')[-1] u = urllib2.urlopen(url) f = open(file_name, 'wb') meta = u.info() file_size = int(meta.getheaders("Content-Length")[0]) print "Downloading: %s Bytes: %s" % (file_name, file_size) file_size_dl = 0 block_sz = 8192 while True: buffer = u.read(block_sz) if not buffer: break file_size_dl += len(buffer) f.write(buffer) status = r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size) status = status + chr(8) * (len(status) + 1) print status, f.close() f = open(file_name, 'rb') csvDict = csv.DictReader(f) print csvDict print f print csvDict.fieldnames print 'File opened.' csvDict.fieldnames = [ field.strip().lower() for field in csvDict.fieldnames ] print 'Field names converted: %s' % str(csvDict.fieldnames) count = 0 existingCount = 0 errorCount = 0 print 'Starting line iteration.' for line in csvDict: if line['year_birthdate'] == '': line['year_birthdate'] = '0' new_year = int(line['year_birthdate']) line['year_birthdate'] = new_year if line['height'] == '': line['height'] = 0 line['height'] = float(line['height']) if line['weight'] == '': line['weight'] = 0 line['weight'] = float(line['weight']) if line['gender'] == '': line['gender'] = 0 elif line['gender'] == '1' or line['gender'] == '2': line['gender'] = int(line['gender']) newUser = NikeUserSport(**line) try: newUser.save() count += 1 except IntegrityError: connection.close() existingCount += 1 except DatabaseError, e: connection.close() # print e errorCount += 1 print '%d rows completed. %d existing rows skipped. Error count: %d \r' % ( count, existingCount, errorCount),