def test__unicode(self): data = Data() data.date = timezone.now() data.value = "100%" data.indicator = "test" self.assertEquals(unicode(data), "test: 100% em 2010-10-10 00:00:00+00:00")
def setUp(self): u = User(username="******") u.save() d = Data() d.save() self.sylva_graph = Graph(name="mygraph", data=d, owner=u) self.sylva_graph.save()
def test__unicode(self): data = Data() data.date = timezone.now() data.value = '100%' data.indicator = 'test' self.assertEquals(unicode(data), 'test: 100% em 2010-10-10 00:00:00+00:00')
def setUp(self): self.u = User(username="******") self.u.save() d = Data() d.save() self.sylva_graph = Graph(name="mygraph", data=d, owner=self.u) self.sylva_graph.save() self.instanceName = "instanceRexster" self.instanceEngine = "engines.gdb.backends.rexster" self.instancePort = "7474" self.instancePath = "db/data"
def setUp(self): user = User.objects.create_user('user', '*****@*****.**', 'pass') user.save() license = License(name='foobar', url='http://foobar.com') license.save() data = Data(name='foobar', pub_date=dt.now(), version=1, user_id=1, license_id=1, is_current=True, is_public=True, is_approved=True, tags='foobar') data.save() p = Preferences(name='default', max_data_size=1024*1024*64) p.save()
def delete_data_from_container(containerid: str = None, data_ids: List[int] = None): """ :param containerid: :param data_ids: :return: """ container = Container.get_object(containerid) DataModel.delete_many(data_ids=data_ids, containerid=containerid) if container.matrix_exists: celery.send_task(RMXWEB_TASKS['integrity_check'], kwargs={'containerid': containerid})
def main(request): responseName = "" setVersion = -1 setCompleted = False if not request.COOKIES.has_key('has_version'): version = random.randint(1, 6) responseName = 'landingpage-' + str(version)+ '.html' setVersion = version if len(Version.objects.filter(version = version)) == 0: newObject = Version(version = version, total = 1, success = 0) newObject.save() else: obj = Version.objects.get(version = version) obj.total += 1 obj.save() else: version = request.COOKIES['has_version'] responseName = 'landingpage-' + str(version)+ '.html' if request.method == 'POST': name = request.POST.get('inputName', '') surname = request.POST.get('inputSurName', '') patronymic = request.POST.get('inputPatronymic', '') phone = request.POST.get('inputPhone', '') email = request.POST.get('inputEmail', '') people = Data(name=name, surname=surname, patronymic=patronymic,phone=phone,email=email) people.save() if request.COOKIES.has_key('has_version'): version = request.COOKIES['has_version'] obj = Version.objects.get(version = version) obj.success += 1 obj.save() setCompleted = True arg = {} if setCompleted or request.COOKIES.has_key('completed'): arg = {'completed':'YES'} else: arg = {'completed':None} response = render_to_response(responseName, arg) if setVersion != -1: response.set_cookie('version', str(setVersion)) if setCompleted: response.set_cookie('completed', 'YES') return response
def add_server(request): data_index_ip_address = request.META.get('REMOTE_ADDR') if request.method == "POST": try: data_operating_system = request.POST['operating_system'] data_ip_address = request.POST['ip_address'] data_system_port = request.POST['system_port'] data_system_owner = request.POST['system_owner'] data_system_username = request.POST['system_username'] data_system_password = request.POST['system_password'] data_system_description = request.POST['system_description'] data_server_name = request.POST['server_name'] except: log = "Add server POST data error|" + str(data_index_ip_address) + "|" + str( datetime.datetime.now()) + " \n" logging.basicConfig(filename='logging/error.log', level=logging.DEBUG) logging.error(log) try: data = Data( operating_system=data_operating_system, ip_address=data_ip_address, system_port=data_system_port, system_owner=data_system_owner, system_username=data_system_username, system_password=data_system_password, system_description=data_system_description, server_name=data_server_name ) data.save() log = "Add server|" + str(data_ip_address)+"|"+str(data_index_ip_address) + "|" + str( datetime.datetime.now()) + " \n" logging.basicConfig(filename='logging/add_server.log', level=logging.DEBUG) logging.info(log) except: log = "Add server to save database error|" + str(data_index_ip_address) + "|" + str( datetime.datetime.now()) + " \n" logging.basicConfig(filename='logging/error.log', level=logging.DEBUG) logging.error(log) add_server_page="Add server:"+data_server_name return HttpResponse(add_server_page) return render(request,'add_server.html')
def process_line(line, logFormat, load): def process_field(log_field): def update_data(data): if "date" in (log_field.field.name).lower(): data.date_field=value if "host" in (log_field.field.name).lower(): data.host_field=value if "status" in (log_field.field.name).lower(): data.status_field=value if "size" in (log_field.field.name).lower(): try: data.size_field=int(value) except: data.size_field=0 if "url" in (log_field.field.name).lower(): data.url_field=value data.url_field_method=value.split(' ')[0] data.url_field_uri=value.split(' ')[1] data.url_field_protocol=value.split(' ')[2] return data value = fields[log_field.position] if log_field.field.type == 2: value = datetime.strptime(fields[log_field.position], logFormat.date_format) tz = pytz.timezone(settings.TIME_ZONE) value = tz.localize(value, is_dst=True) update_data(data) fields = re.match( logFormat.regex, line).groups() log_fields = LogField.objects.filter(log_format=logFormat.pk).order_by("position") data = Data(load=load) for log_field in log_fields: process_field(log_field) data.save()
def test_create_data_set(self): d = Data(name = 'test_data_set', user=User.objects.get(username='******'), license=License.objects.get(name='foobar')) d.create_slug() d.attach_file(File(open('fixtures/breastcancer.txt', 'r'))) d.save() self.assertEqual(1, len(Data.objects.filter(name='test_data_set')))
async def parse_page(self, session: aiohttp.ClientSession, page_num: int) -> List[Data]: """ парсит отдельную страницу вытаскивая id и ссылку на закупку """ param = self.params.copy() param['pageNumber'] = page_num orders = [] data = await self._get_request(session, self.search_link, params=param) doc = document_fromstring(data) for link in doc.xpath( '//div[@class="registry-entry__header-mid__number"]/a'): order_id = self._tender_id_handler( self._normalizer(link.xpath('./text()')[0])) order_link = self.main_link + link.xpath('./@href')[0] tender = Data() tender.id = order_id tender.tender_link = order_link orders.append(tender) return orders
def get(self, request, format=None): """Retrieves a list of all existing containers.""" containers = Container.objects.all().order_by('-created') data_seed = Data.filter_seed_data([_.pk for _ in containers]) serialiser = SerialiserFactory().get_serialiser('container_csv') serialiser = serialiser(data={ 'container': containers, 'dataset': data_seed }) zip_name = serialiser.get_zip_name('Containers-List') resp = HttpResponse(serialiser.get_value(), content_type='application/force-download') resp['Content-Disposition'] = 'attachment; filename="%s"' % zip_name return resp
def make_random_data(): new_data = Data() new_data.dust25 = random.randint(5, 200) new_data.dust100 = random.randint(5, 20) new_data.discomfort_index = random.randint(20, 60) new_data.co2 = random.randint(400, 1200) new_data.temp = random.randint(180, 230) / 10 new_data.humid = random.randint(40, 50) new_data.device_id = str(random.randint(100000, 999999)) + "-" \ + str(random.randint(100000, 999999)) + "-" + \ str(random.randint(100000, 999999)) new_data.save()
async def parse_ea44(self, session: aiohttp.ClientSession, order: Data) -> Data: """ парсит данные одной закупки """ # проверка на наличие записи в базе data = self.db_session.query(Data).filter(Data.id == order.id).all() if data: return data[0] text = await self._get_request(session, order.tender_link) order_document = document_fromstring(text) # парсим главную информацию о закупке - номер цена заказчик дата card_info_container = order_document.cssselect('.cardMainInfo')[0] tender_id = card_info_container.cssselect( '.cardMainInfo__purchaseLink') if not tender_id: tender_id = '' else: tender_id = self._normalizer(tender_id[0].text_content()) tender_object = card_info_container.xpath( './div[1]/div[2]/div[1]/span[2]') if not tender_object: tender_object = '' else: tender_object = self._normalizer(tender_object[0].text_content()) customer = card_info_container.xpath('./div[1]/div[2]/div[2]/span[2]') if not customer: customer = '' else: customer = self._normalizer(customer[0].text_content()) tender_price = card_info_container.cssselect('.cost') if not tender_price: tender_price = '' else: tender_price = self._normalizer(tender_price[0].text_content()) tender_date = card_info_container.xpath( './div[2]/div[2]/div[1]/div[1]/span[2]') if not tender_date: tender_date = '' else: tender_date = self._normalizer(tender_date[0].text_content()) # общая информация о закупке - адресс электронной площадки и обьект закупки general_information_container = order_document.xpath( '//div[@class="wrapper"]/div[2]') tender_adress = general_information_container[0].xpath( './/div[@class="col"]/section[3]/span[2]') if not tender_adress: tender_adress = '' else: tender_adress = self._normalizer(tender_adress[0].text_content()) # условия контракта condition_container = self.get_cotract_conditions_container( order_document.xpath('//div[@id="custReqNoticeTable"]/div')) if condition_container is not None: tender_delivery_adress = condition_container.xpath( './/div[@class="col"]/section[2]/span[2]') if not tender_delivery_adress: tender_delivery_adress = '' else: tender_delivery_adress = self._normalizer( tender_delivery_adress[0].text_content()) tender_term = condition_container.xpath( './/div[@class="row"]/section[3]/span[2]') if not tender_term: tender_term = '' else: tender_term = self._normalizer(tender_term[0].text_content()) else: tender_delivery_adress = '' tender_term = '' # парсинг информации о обьекте закупки tender_object_info = self.parse_tender_object_info(order_document) # парсинг победителя try: winner = await self.parse_tender_winner(session, order.tender_link) except Exception: winner = [] if len(winner) < 3: winner = ['', '', ''] # парсинг ссылок документов term_document_link = order.tender_link.replace('common-info', 'documents') term_document_data = await self._get_request(session, term_document_link) term_document_links = document_fromstring(term_document_data).xpath( '//span[@class="section__value"]/a[@title]/@href') order.tender_object = tender_object order.customer = customer order.tender_price = self._tender_price_handler(tender_price) order.tender_date = self._tender_date_handler(tender_date) order.tender_adress = tender_adress order.tender_delivery = tender_delivery_adress order.tender_term = tender_term for object_info in self._handle_tender_objects(tender_object_info): order.objects.append(object_info) for document_link_data in term_document_links: tender_link = TenderLinks() tender_link.link = document_link_data tender_link.data_id = order.id order.document_links.append(tender_link) order.winner.append( Winners(name=winner[0], position=winner[1], price=winner[2])) order.type = 'fz44' return order
def db_handler(self, id, data): session = self.session_maker() db_data = Data() db_data.id = id db_data.tender_price = self._tender_price_handler(data['tender_price']) db_data.type = data['type'] db_data.tender_date = self._tender_date_handler(data['tender_date']) db_data.tender_object = data['tender_object'] db_data.customer = data['customer'] db_data.tender_adress = data['tender_adress'] db_data.tender_delivery = data['tender_delivery'] db_data.tender_terms = data['tender_term'] db_data.document_links = '\n'.join(data['document_links']) db_data.tender_link = data['link'] winner = Winners() winner.data_id = id winner.name = data['tender_winner'][0] winner.position = data['tender_winner'][1] winner.price = data['tender_winner'][2] db_data.winner.append(winner) for i in data['tender_object_info'][1:]: if len(i) == 6: data = Objects() data.position = i[0] data.name = i[1] data.unit = i[2] data.quantity = i[3] data.unit_price = i[4] data.price = i[5] db_data.objects.append(data) session.add(db_data) session.commit() db_data.winner = [winner] session.close() return db_data