def test__unicode(self):
data = Data()
data.date = timezone.now()
data.value = "100%"
data.indicator = "test"
self.assertEquals(unicode(data), "test: 100% em 2010-10-10 00:00:00+00:00")
def setUp(self):
u = User(username="******")
u.save()
d = Data()
d.save()
self.sylva_graph = Graph(name="mygraph", data=d, owner=u)
self.sylva_graph.save()
def test__unicode(self):
data = Data()
data.date = timezone.now()
data.value = '100%'
data.indicator = 'test'
self.assertEquals(unicode(data),
'test: 100% em 2010-10-10 00:00:00+00:00')
def setUp(self):
self.u = User(username="******")
self.u.save()
d = Data()
d.save()
self.sylva_graph = Graph(name="mygraph", data=d, owner=self.u)
self.sylva_graph.save()
self.instanceName = "instanceRexster"
self.instanceEngine = "engines.gdb.backends.rexster"
self.instancePort = "7474"
self.instancePath = "db/data"
def setUp(self):
user = User.objects.create_user('user', '*****@*****.**', 'pass')
user.save()
license = License(name='foobar', url='http://foobar.com')
license.save()
data = Data(name='foobar',
pub_date=dt.now(), version=1, user_id=1, license_id=1,
is_current=True, is_public=True, is_approved=True,
tags='foobar')
data.save()
p = Preferences(name='default', max_data_size=1024*1024*64)
p.save()
def delete_data_from_container(containerid: str = None,
data_ids: List[int] = None):
"""
:param containerid:
:param data_ids:
:return:
"""
container = Container.get_object(containerid)
DataModel.delete_many(data_ids=data_ids, containerid=containerid)
if container.matrix_exists:
celery.send_task(RMXWEB_TASKS['integrity_check'],
kwargs={'containerid': containerid})
def main(request):
responseName = ""
setVersion = -1
setCompleted = False
if not request.COOKIES.has_key('has_version'):
version = random.randint(1, 6)
responseName = 'landingpage-' + str(version)+ '.html'
setVersion = version
if len(Version.objects.filter(version = version)) == 0:
newObject = Version(version = version, total = 1, success = 0)
newObject.save()
else:
obj = Version.objects.get(version = version)
obj.total += 1
obj.save()
else:
version = request.COOKIES['has_version']
responseName = 'landingpage-' + str(version)+ '.html'
if request.method == 'POST':
name = request.POST.get('inputName', '')
surname = request.POST.get('inputSurName', '')
patronymic = request.POST.get('inputPatronymic', '')
phone = request.POST.get('inputPhone', '')
email = request.POST.get('inputEmail', '')
people = Data(name=name, surname=surname, patronymic=patronymic,phone=phone,email=email)
people.save()
if request.COOKIES.has_key('has_version'):
version = request.COOKIES['has_version']
obj = Version.objects.get(version = version)
obj.success += 1
obj.save()
setCompleted = True
arg = {}
if setCompleted or request.COOKIES.has_key('completed'):
arg = {'completed':'YES'}
else:
arg = {'completed':None}
response = render_to_response(responseName, arg)
if setVersion != -1:
response.set_cookie('version', str(setVersion))
if setCompleted:
response.set_cookie('completed', 'YES')
return response
def add_server(request):
data_index_ip_address = request.META.get('REMOTE_ADDR')
if request.method == "POST":
try:
data_operating_system = request.POST['operating_system']
data_ip_address = request.POST['ip_address']
data_system_port = request.POST['system_port']
data_system_owner = request.POST['system_owner']
data_system_username = request.POST['system_username']
data_system_password = request.POST['system_password']
data_system_description = request.POST['system_description']
data_server_name = request.POST['server_name']
except:
log = "Add server POST data error|" + str(data_index_ip_address) + "|" + str(
datetime.datetime.now()) + " \n"
logging.basicConfig(filename='logging/error.log', level=logging.DEBUG)
logging.error(log)
try:
data = Data(
operating_system=data_operating_system,
ip_address=data_ip_address,
system_port=data_system_port,
system_owner=data_system_owner,
system_username=data_system_username,
system_password=data_system_password,
system_description=data_system_description,
server_name=data_server_name
)
data.save()
log = "Add server|" + str(data_ip_address)+"|"+str(data_index_ip_address) + "|" + str(
datetime.datetime.now()) + " \n"
logging.basicConfig(filename='logging/add_server.log', level=logging.DEBUG)
logging.info(log)
except:
log = "Add server to save database error|" + str(data_index_ip_address) + "|" + str(
datetime.datetime.now()) + " \n"
logging.basicConfig(filename='logging/error.log', level=logging.DEBUG)
logging.error(log)
add_server_page="Add server:"+data_server_name
return HttpResponse(add_server_page)
return render(request,'add_server.html')
def process_line(line, logFormat, load):
def process_field(log_field):
def update_data(data):
if "date" in (log_field.field.name).lower():
data.date_field=value
if "host" in (log_field.field.name).lower():
data.host_field=value
if "status" in (log_field.field.name).lower():
data.status_field=value
if "size" in (log_field.field.name).lower():
try:
data.size_field=int(value)
except:
data.size_field=0
if "url" in (log_field.field.name).lower():
data.url_field=value
data.url_field_method=value.split(' ')[0]
data.url_field_uri=value.split(' ')[1]
data.url_field_protocol=value.split(' ')[2]
return data
value = fields[log_field.position]
if log_field.field.type == 2:
value = datetime.strptime(fields[log_field.position], logFormat.date_format)
tz = pytz.timezone(settings.TIME_ZONE)
value = tz.localize(value, is_dst=True)
update_data(data)
fields = re.match( logFormat.regex, line).groups()
log_fields = LogField.objects.filter(log_format=logFormat.pk).order_by("position")
data = Data(load=load)
for log_field in log_fields:
process_field(log_field)
data.save()
def test_create_data_set(self):
d = Data(name = 'test_data_set',
user=User.objects.get(username='******'),
license=License.objects.get(name='foobar'))
d.create_slug()
d.attach_file(File(open('fixtures/breastcancer.txt', 'r')))
d.save()
self.assertEqual(1, len(Data.objects.filter(name='test_data_set')))
async def parse_page(self, session: aiohttp.ClientSession,
page_num: int) -> List[Data]:
""" парсит отдельную страницу вытаскивая id и ссылку на закупку """
param = self.params.copy()
param['pageNumber'] = page_num
orders = []
data = await self._get_request(session, self.search_link, params=param)
doc = document_fromstring(data)
for link in doc.xpath(
'//div[@class="registry-entry__header-mid__number"]/a'):
order_id = self._tender_id_handler(
self._normalizer(link.xpath('./text()')[0]))
order_link = self.main_link + link.xpath('./@href')[0]
tender = Data()
tender.id = order_id
tender.tender_link = order_link
orders.append(tender)
return orders
def get(self, request, format=None):
"""Retrieves a list of all existing containers."""
containers = Container.objects.all().order_by('-created')
data_seed = Data.filter_seed_data([_.pk for _ in containers])
serialiser = SerialiserFactory().get_serialiser('container_csv')
serialiser = serialiser(data={
'container': containers,
'dataset': data_seed
})
zip_name = serialiser.get_zip_name('Containers-List')
resp = HttpResponse(serialiser.get_value(),
content_type='application/force-download')
resp['Content-Disposition'] = 'attachment; filename="%s"' % zip_name
return resp
def make_random_data():
new_data = Data()
new_data.dust25 = random.randint(5, 200)
new_data.dust100 = random.randint(5, 20)
new_data.discomfort_index = random.randint(20, 60)
new_data.co2 = random.randint(400, 1200)
new_data.temp = random.randint(180, 230) / 10
new_data.humid = random.randint(40, 50)
new_data.device_id = str(random.randint(100000, 999999)) + "-" \
+ str(random.randint(100000, 999999)) + "-" + \
str(random.randint(100000, 999999))
new_data.save()
async def parse_ea44(self, session: aiohttp.ClientSession,
order: Data) -> Data:
""" парсит данные одной закупки """
# проверка на наличие записи в базе
data = self.db_session.query(Data).filter(Data.id == order.id).all()
if data:
return data[0]
text = await self._get_request(session, order.tender_link)
order_document = document_fromstring(text)
# парсим главную информацию о закупке - номер цена заказчик дата
card_info_container = order_document.cssselect('.cardMainInfo')[0]
tender_id = card_info_container.cssselect(
'.cardMainInfo__purchaseLink')
if not tender_id:
tender_id = ''
else:
tender_id = self._normalizer(tender_id[0].text_content())
tender_object = card_info_container.xpath(
'./div[1]/div[2]/div[1]/span[2]')
if not tender_object:
tender_object = ''
else:
tender_object = self._normalizer(tender_object[0].text_content())
customer = card_info_container.xpath('./div[1]/div[2]/div[2]/span[2]')
if not customer:
customer = ''
else:
customer = self._normalizer(customer[0].text_content())
tender_price = card_info_container.cssselect('.cost')
if not tender_price:
tender_price = ''
else:
tender_price = self._normalizer(tender_price[0].text_content())
tender_date = card_info_container.xpath(
'./div[2]/div[2]/div[1]/div[1]/span[2]')
if not tender_date:
tender_date = ''
else:
tender_date = self._normalizer(tender_date[0].text_content())
# общая информация о закупке - адресс электронной площадки и обьект закупки
general_information_container = order_document.xpath(
'//div[@class="wrapper"]/div[2]')
tender_adress = general_information_container[0].xpath(
'.//div[@class="col"]/section[3]/span[2]')
if not tender_adress:
tender_adress = ''
else:
tender_adress = self._normalizer(tender_adress[0].text_content())
# условия контракта
condition_container = self.get_cotract_conditions_container(
order_document.xpath('//div[@id="custReqNoticeTable"]/div'))
if condition_container is not None:
tender_delivery_adress = condition_container.xpath(
'.//div[@class="col"]/section[2]/span[2]')
if not tender_delivery_adress:
tender_delivery_adress = ''
else:
tender_delivery_adress = self._normalizer(
tender_delivery_adress[0].text_content())
tender_term = condition_container.xpath(
'.//div[@class="row"]/section[3]/span[2]')
if not tender_term:
tender_term = ''
else:
tender_term = self._normalizer(tender_term[0].text_content())
else:
tender_delivery_adress = ''
tender_term = ''
# парсинг информации о обьекте закупки
tender_object_info = self.parse_tender_object_info(order_document)
# парсинг победителя
try:
winner = await self.parse_tender_winner(session, order.tender_link)
except Exception:
winner = []
if len(winner) < 3:
winner = ['', '', '']
# парсинг ссылок документов
term_document_link = order.tender_link.replace('common-info',
'documents')
term_document_data = await self._get_request(session,
term_document_link)
term_document_links = document_fromstring(term_document_data).xpath(
'//span[@class="section__value"]/a[@title]/@href')
order.tender_object = tender_object
order.customer = customer
order.tender_price = self._tender_price_handler(tender_price)
order.tender_date = self._tender_date_handler(tender_date)
order.tender_adress = tender_adress
order.tender_delivery = tender_delivery_adress
order.tender_term = tender_term
for object_info in self._handle_tender_objects(tender_object_info):
order.objects.append(object_info)
for document_link_data in term_document_links:
tender_link = TenderLinks()
tender_link.link = document_link_data
tender_link.data_id = order.id
order.document_links.append(tender_link)
order.winner.append(
Winners(name=winner[0], position=winner[1], price=winner[2]))
order.type = 'fz44'
return order
def db_handler(self, id, data):
session = self.session_maker()
db_data = Data()
db_data.id = id
db_data.tender_price = self._tender_price_handler(data['tender_price'])
db_data.type = data['type']
db_data.tender_date = self._tender_date_handler(data['tender_date'])
db_data.tender_object = data['tender_object']
db_data.customer = data['customer']
db_data.tender_adress = data['tender_adress']
db_data.tender_delivery = data['tender_delivery']
db_data.tender_terms = data['tender_term']
db_data.document_links = '\n'.join(data['document_links'])
db_data.tender_link = data['link']
winner = Winners()
winner.data_id = id
winner.name = data['tender_winner'][0]
winner.position = data['tender_winner'][1]
winner.price = data['tender_winner'][2]
db_data.winner.append(winner)
for i in data['tender_object_info'][1:]:
if len(i) == 6:
data = Objects()
data.position = i[0]
data.name = i[1]
data.unit = i[2]
data.quantity = i[3]
data.unit_price = i[4]
data.price = i[5]
db_data.objects.append(data)
session.add(db_data)
session.commit()
db_data.winner = [winner]
session.close()
return db_data
