def do_chart(request, slug):
import re
from django.http import HttpResponse
from mxhelpers import get_rtt_dataset
mx_obj = Mailserver.objects.get(slug=slug)
graph_dataset = get_rtt_dataset(mx_obj, request.GET.get('range'))
#rtt_labels = graph_dataset['rtt_labels']
data_temp = "t:%s|%s|%s" % (graph_dataset['sg_rtt_dataset'],
graph_dataset['us_rtt_dataset'],
graph_dataset['uk_rtt_dataset'])
#au_rtt_list = re.sub("[\[\]]", '', str(graph_dataset['au_rtt_dataset']))
#au_rtt_dataset = graph_dataset['au_rtt_dataset']
#data_points = "t:%s" % re.sub("[\[\]\' ']", '', str(au_rtt_dataset))
data_points = re.sub("[\[\]\' ']", '', str(data_temp))
import urllib
import urllib2
import time
url = 'http://chart.apis.google.com/chart?'
#data_points = 't:40,20,50,20,100|50,20,30,40,50'
values = {
'cht':
'lc',
'chs':
'700x200',
'chds':
'0,%s' % graph_dataset['max_num'],
'chco':
'96513c,008000,2865cc',
'chdlp':
'b',
'chxt':
'x,y',
'chxl':
'0:|%s|1:|0|%s' %
(graph_dataset['rtt_labels'], graph_dataset['rtt_y_labels']),
'chdl':
'Singapore|United Status|United Kingdom',
'chd':
data_points
}
data = urllib.urlencode(values)
req = urllib2.Request(url, data)
#url_resp = urllib2.urlopen(req)
#response = url_resp.read()
from django.core.files import File
from django.core.files.temp import NamedTemporaryFile
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(urllib2.urlopen(req).read())
img_temp.flush()
response = HttpResponse(File(img_temp), mimetype="image/png")
return response
def create_eks_cluster(challenge):
"""
Called when Challenge is approved by the EvalAI admin
calls the create_eks_nodegroup function
Arguments:
sender {type} -- model field called the post hook
instance {<class 'django.db.models.query.QuerySet'>} -- instance of the model calling the post hook
"""
from .models import ChallengeEvaluationCluster
for obj in serializers.deserialize("json", challenge):
challenge_obj = obj.object
cluster_name = "{0}-cluster".format(challenge_obj.title.replace(" ", "-"))
if challenge_obj.approved_by_admin and challenge_obj.is_docker_based:
client = get_boto3_client("eks", aws_keys)
try:
response = client.create_cluster(
name=cluster_name,
version="1.15",
roleArn=settings.EKS_CLUSTER_ROLE_ARN,
resourcesVpcConfig={
"subnetIds": [VPC_DICT["SUBNET_1"], VPC_DICT["SUBNET_2"]],
"securityGroupIds": [VPC_DICT["SUBNET_SECURITY_GROUP"]],
},
)
waiter = client.get_waiter("cluster_active")
waiter.wait(name=cluster_name)
# creating kubeconfig
cluster = client.describe_cluster(name=cluster_name)
cluster_cert = cluster["cluster"]["certificateAuthority"]["data"]
cluster_ep = cluster["cluster"]["endpoint"]
cluster_config = {
"apiVersion": "v1",
"kind": "Config",
"clusters": [
{
"cluster": {
"server": str(cluster_ep),
"certificate-authority-data": str(cluster_cert),
},
"name": "kubernetes",
}
],
"contexts": [
{
"context": {"cluster": "kubernetes", "user": "******"},
"name": "aws",
}
],
"current-context": "aws",
"preferences": {},
"users": [
{
"name": "aws",
"user": {
"exec": {
"apiVersion": "client.authentication.k8s.io/v1alpha1",
"command": "heptio-authenticator-aws",
"args": ["token", "-i", cluster_name],
}
},
}
],
}
# Write in YAML.
config_text = yaml.dump(cluster_config, default_flow_style=False)
config_file = NamedTemporaryFile(delete=True)
config_file.write(config_text.encode())
ChallengeEvaluationCluster.objects.create(
challenge=challenge_obj,
name=cluster_name,
cluster_endpoint=cluster_ep,
cluster_ssl=cluster_cert,
)
# Creating nodegroup
create_eks_nodegroup.delay(challenge, cluster_name)
return response
except ClientError as e:
logger.exception(e)
return
def process_locale_dir(self, locale_dir, files):
"""
Extract translatable literals from the specified files, creating or
updating the POT file for a given locale directory.
Use the xgettext GNU gettext utility.
"""
build_files = []
for translatable in files:
if self.verbosity > 1:
self.stdout.write('processing file %s in %s\n' % (
translatable.file, translatable.dirpath
))
if self.domain not in ('djangojs', 'django'):
continue
build_file = self.build_file_class(self, self.domain, translatable)
try:
build_file.preprocess()
except UnicodeDecodeError as e:
self.stdout.write(
'UnicodeDecodeError: skipped file %s in %s (reason: %s)' % (
translatable.file, translatable.dirpath, e,
)
)
continue
build_files.append(build_file)
if self.domain == 'djangojs':
is_templatized = build_file.is_templatized
args = [
'xgettext',
'-d', self.domain,
'--language=%s' % ('C' if is_templatized else 'JavaScript',),
'--keyword=gettext_noop',
'--keyword=gettext_lazy',
'--keyword=ngettext_lazy:1,2',
'--keyword=pgettext:1c,2',
'--keyword=npgettext:1c,2,3',
'--output=-',
]
elif self.domain == 'django':
args = [
'xgettext',
'-d', self.domain,
'--language=Python',
'--keyword=gettext_noop',
'--keyword=gettext_lazy',
'--keyword=ngettext_lazy:1,2',
'--keyword=ugettext_noop',
'--keyword=ugettext_lazy',
'--keyword=ungettext_lazy:1,2',
'--keyword=pgettext:1c,2',
'--keyword=npgettext:1c,2,3',
'--keyword=pgettext_lazy:1c,2',
'--keyword=npgettext_lazy:1c,2,3',
'--output=-',
]
else:
return
input_files = [bf.work_path for bf in build_files]
with NamedTemporaryFile(mode='w+') as input_files_list:
input_files_list.write('\n'.join(input_files))
input_files_list.flush()
args.extend(['--files-from', input_files_list.name])
args.extend(self.xgettext_options)
msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
for build_file in build_files:
build_file.cleanup()
raise CommandError(
'errors happened while running xgettext on %s\n%s' %
('\n'.join(input_files), errors)
)
elif self.verbosity > 0:
# Print warnings
self.stdout.write(errors)
if msgs:
if locale_dir is NO_LOCALE_DIR:
file_path = os.path.normpath(build_files[0].path)
raise CommandError(
'Unable to find a locale path to store translations for '
'file %s' % file_path
)
for build_file in build_files:
msgs = build_file.postprocess_messages(msgs)
potfile = os.path.join(locale_dir, '%s.pot' % self.domain)
write_pot_file(potfile, msgs)
for build_file in build_files:
build_file.cleanup()
def handle(self, *args, **options):
# Get default profile image
image_url = '''http://cdn.scahw.com.au/cdn-1cfaa3a9e77a520/
imagevaultfiles/id_301664/cf_3/random-animals-16.jpg'''
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(urlopen(image_url).read())
img_temp.flush()
pic = File(img_temp)
# Get default file to use for resume
resume_url = '''http://writing.colostate.edu/guides/documents/resume/
functionalSample.pdf'''
resume_temp = NamedTemporaryFile(delete=True)
resume_temp.write(urlopen(resume_url).read())
resume_temp.flush()
resume = File(resume_temp)
# Demo hobbies to be used in following functions
demo_hobbies = [
Hobby.objects.all().order_by('?').first(),
Hobby.objects.all().order_by('?').first(),
Hobby.objects.all().order_by('?').first(),
Hobby.objects.all().order_by('?').first(),
Hobby.objects.all().order_by('?').first(),
]
# Create demo student user
if not MyUser.objects.filter(email='*****@*****.**').exists():
student_demo = MyUser.objects.create_user(
email='*****@*****.**',
first_name='Student',
last_name='Sample',
password='******',
gpa=3.45,
degree_earned=MyUser.BACHELOR,
year=MyUser.SENIOR,
opp_sought=MyUser.FULL_TIME,
)
student_demo.profile_pic.save("image_%s" % student_demo.username,
pic),
student_demo.resume.save("resume_%s" % student_demo.username,
resume),
student_demo.is_confirmed = True
student_demo.account_type = MyUser.STUDENT
student_demo.video = 'https://www.youtube.com/watch?v=_OBlgSz8sSM'
student_demo.undergrad_uni = School.objects.get(
name='Brown University')
student_demo.undergrad_degree.add(
Degree.objects.get(name='Business'))
student_demo.save()
for hobby in demo_hobbies:
student_demo.hobbies.add(hobby)
self.stdout.write(
self.style.WARNING('Created demo student account.'))
# Create demo employer user
if not MyUser.objects.filter(email='*****@*****.**').exists():
employer_demo = MyUser.objects.create_user(
email='*****@*****.**',
first_name='Employer',
last_name='Sample',
password='******')
employer_demo.is_confirmed = True
employer_demo.account_type = MyUser.EMPLOYER
employer_demo.save()
self.stdout.write(
self.style.WARNING('Created demo employer account.'))
# Create users
users = [['John', 'Doe'], ['Bill', 'Clarkson'], ['Jessica', 'Hall'],
['Franklin', 'Rose'], ['Bobby', 'Collins'],
['Fred', 'Flinstone'], ['Blake', 'Casper'],
['Marissa', 'Smiles'], ['Tyler', 'Simm'],
['Gina', 'Zentenial'], ['Michelle', 'Gregs'],
['Oscar', 'Behaves'], ['Heather', 'Hoolihan'],
['Scott', 'Dragonile'], ['Charlie', 'Bitfinger'],
['Pryia', 'Havalopolous'], ['Chris', 'Wildwood'],
['Jonathan', 'Newguinea'], ['Anne', 'Hathaway'],
['Brooke', 'Orchard']]
if not MyUser.objects.filter(first_name=users[0][0]).exists():
for user in users:
user = MyUser.objects.create_user(
email='{}.{}@{}'.format(
user[0], user[1],
School.objects.all().order_by(
'?').first().email).lower(),
first_name=user[0],
last_name=user[1],
password='******',
gpa=randint(int(2), int(100 * 4)) / 100.0,
degree_earned=random.choice(
[x[0] for x in MyUser.DEGREE_TYPES]),
year=random.choice([x[0] for x in MyUser.YEAR_TYPES]),
opp_sought=random.choice(
[x[0] for x in MyUser.OPPORTUNITY_TYPES]))
user.is_confirmed = True
user.account_type = MyUser.STUDENT
user.profile_pic.save("image_%s" % user.username, pic),
user.resume.save("resume_%s" % user.username, resume),
user.undergrad_uni = School.objects.all().order_by('?').first()
user.undergrad_degree.add(
Degree.objects.all().order_by('?').first())
user.save()
for hobby in demo_hobbies:
user.hobbies.add(hobby)
self.stdout.write(
self.style.WARNING('Created demo student accounts.'))
# Create companies
company_owners = [['Jeff', 'Bezos'], ['Pablo', 'Padre'],
['Jaime', 'Windell'], ['Naome', 'Watts']]
if not MyUser.objects.filter(first_name=company_owners[0][0]).exists():
for user in company_owners:
user = MyUser.objects.create_user(
email='{}.{}@demo.com'.format(user[0], user[1]).lower(),
first_name=user[0],
last_name=user[1],
password='******')
user.is_confirmed = True
user.account_type = MyUser.EMPLOYER
user.save()
self.stdout.write(
self.style.WARNING('Created demo employer owner accounts.'))
companies = [
['Goldman Sachs',
MyUser.objects.get(first_name='Jeff')],
['JPMorgan Chase',
MyUser.objects.get(first_name='Pablo')],
['Morgan Stanley',
MyUser.objects.get(first_name='Jaime')],
['American Express',
MyUser.objects.get(first_name='Naome')],
['KPMG', MyUser.objects.get(first_name='Employer')],
]
if not Company.objects.filter(name=companies[0][0]).exists():
for company in companies:
Company.objects.create(user=company[1], name=company[0])
self.stdout.write(
self.style.WARNING('Created demo employer accounts.'))
# Create jobs
description = '''Affert officiis consequat ut his. Vix cu ferri
nostrum contentiones, in pri partem principes, pri id aliquid
pericula. Nam atqui intellegat at. Et soleat perfecto mei, elitr
abhorreant his an. Ex soleat habemus splendide mei, duo eu suas iisque,
id eam justo argumentum. In sea cetero erroribus vituperatoribus.
Dolorum senserit ad pri, no est nusquam definitiones.
Has ipsum tincidunt ne. Eu mea aperiri euismod, vix in nominati
inimicus. At simul adipiscing nec, dolore laboramus pro no, sea tale
fierent ne. An corpora detracto corrumpit pri, epicurei intellegam quo
in, dicit verterem id sit'''
start = datetime.now()
end = start + timedelta(days=21)
contact_email = '*****@*****.**'
jobs = [['Sales Associate', 'New York, NY'],
['IT Consultant', 'Hoboken, NJ'],
['Event Manager', 'Los Angeles, CA'],
['Senior Director', 'New York, NY'], ['EVP', 'Boston, MA'],
['Software Developer', 'Menlo Park, CA'],
['Marketing Associate', 'La Jolla, CA']]
if not Job.objects.filter(title=jobs[0][0]).exists():
for job in jobs:
company = Company.objects.order_by('?').first()
Job.objects.create(
user=company.user,
company=company,
title=job[0],
contact_email=contact_email,
description=description,
location=job[1],
list_date_start=start,
list_date_end=end,
)
self.stdout.write(self.style.WARNING('Created demo job listings.'))
# Create applicants
if not Applicant.objects.filter(user=MyUser.objects.get(
first_name=users[0][0])).exists():
for applicant in users:
_user = MyUser.objects.get(first_name=applicant[0])
Applicant.objects.create(user=_user,
resume=settings.STATIC_URL +
'img/default-profile-pic.jpg',
email=_user.email)
self.stdout.write(self.style.WARNING('Created demo applicants.'))
# Add applicants to jobs
for applicant in Applicant.objects.all():
job = Job.objects.all().order_by('?').first()
job.applicants.add(applicant)
self.stdout.write(
self.style.WARNING('Added applicants to demo job listings.'))
self.stdout.write(
self.style.SUCCESS('Successfully generated all sample data.'))
def test_international_homepage_serializer(rf, international_root_page, image):
home_page = InternationalHomePageFactory(
slug='international',
parent=
international_root_page, # This is tautological, but irrelevant here
)
home_page.hero_subtitle = "THIS LEGACY FIELD SHOULD NOT BE USED"
fake_video = make_test_video(duration=70,
transcript='Test transcript note')
with NamedTemporaryFile(delete=True) as img_tmp:
fake_video.thumbnail.save('test.jpg', File(img_tmp))
fake_video.save()
home_page.hero_video = fake_video
home_page.hero_video.save()
home_page.homepage_link_panels = [
(
'link_panel',
{
'title': 'panel one',
'supporting_text': 'panel one supporting text',
'link': {
'internal_link':
home_page, # Circular link but doesn't matter in this test
}
}),
('link_panel', {
'title': 'panel two',
'supporting_text': 'panel two supporting text',
'link': {
'external_link': 'http://example.com/two/',
}
}),
]
home_page.save()
assert home_page.hero_subtitle is not None
serializer = InternationalHomePageSerializer(
instance=home_page, context={'request': rf.get('/')})
assert serializer.data['hero_title'] == home_page.hero_title
homepage_link_panels = serializer.data['homepage_link_panels']
assert homepage_link_panels[0]['type'] == 'link_panel'
assert homepage_link_panels[0]['value'] == {
'title': 'panel one',
'supporting_text': 'panel one supporting text',
'link': home_page.url,
}
assert homepage_link_panels[1]['type'] == 'link_panel'
assert homepage_link_panels[1]['value'] == {
'title': 'panel two',
'supporting_text': 'panel two supporting text',
'link': 'http://example.com/two/',
}
assert serializer.data['hero_video']['title'] == 'Test file'
assert serializer.data['hero_video'][
'transcript'] == 'Test transcript note'
assert serializer.data['hero_video'][
'thumbnail'] == fake_video.thumbnail.url
assert serializer.data['hero_video']['sources'][0]['src'] == fake_video.url
# confirm the legacy fields are not exposed:
for example_field_name in [
'hero_subtitle',
'hero_cta_text',
'hero_cta_link',
'hero_image',
'brexit_banner_text',
'invest_title',
'invest_content',
'invest_image',
# there are more, but these are useful smoke tests
'case_study_image',
'case_study_title',
'case_study_text',
'case_study_cta_text',
'case_study_cta_link',
]:
assert example_field_name not in serializer.data
def generate_export(export_type,
extension,
username,
id_string,
export_id=None,
filter_query=None,
group_delimiter='/',
split_select_multiples=True,
binary_select_multiples=False):
"""
Create appropriate export object given the export type
"""
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
Export.ANALYSER_EXPORT: 'to_analyser_export'
}
xform = XForm.objects.get(user__username__iexact=username,
id_string__exact=id_string)
# query mongo for the cursor
records = query_mongo(username, id_string, filter_query)
export_builder = ExportBuilder()
export_builder.GROUP_DELIMITER = group_delimiter
export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples
export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples
export_builder.set_survey(xform.data_dictionary().survey)
prefix = slugify('{}_export__{}__{}'.format(export_type, username,
id_string))
temp_file = NamedTemporaryFile(prefix=prefix, suffix=("." + extension))
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
func.__call__(temp_file.name, records, username, id_string, filter_query)
# generate filename
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
if export_type == Export.ANALYSER_EXPORT:
# Analyser exports should be distinguished by more than just their file extension.
basename = '{}_ANALYSER_{}'.format(
id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(username, 'exports', id_string, export_type,
filename)
# TODO: if s3 storage, make private - how will we protect local storage??
storage = get_storage_class()()
# seek to the beginning as required by storage classes
temp_file.seek(0)
export_filename = storage.save(file_path, File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
if export_id:
export = Export.objects.get(id=export_id)
else:
export = Export(xform=xform, export_type=export_type)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# dont persist exports that have a filter
if filter_query is None:
export.save()
return export
def import_entries(self, feed_entries):
"""Import entries"""
for feed_entry in feed_entries:
self.write_out('> %s... ' % feed_entry.title)
if feed_entry.get('published_parsed'):
creation_date = datetime(*feed_entry.published_parsed[:6])
if settings.USE_TZ:
creation_date = timezone.make_aware(
creation_date, timezone.utc)
else:
creation_date = timezone.now()
slug = slugify(feed_entry.title)[:255]
if Entry.objects.filter(creation_date__year=creation_date.year,
creation_date__month=creation_date.month,
creation_date__day=creation_date.day,
slug=slug):
self.write_out(
self.style.NOTICE('SKIPPED (already imported)\n'))
continue
categories = self.import_categories(feed_entry)
entry_dict = {
'title': feed_entry.title[:255],
'content': feed_entry.description,
'excerpt': feed_entry.get('summary'),
'status': PUBLISHED,
'creation_date': creation_date,
'start_publication': creation_date,
'last_update': timezone.now(),
'slug': slug
}
if not entry_dict['excerpt'] and self.auto_excerpt:
entry_dict['excerpt'] = Truncator(
strip_tags(feed_entry.description)).words(50)
if self.tags:
entry_dict['tags'] = self.import_tags(categories)
entry = Entry(**entry_dict)
entry.save()
entry.categories.add(*categories)
entry.sites.add(self.SITE)
if self.image_enclosure:
for enclosure in feed_entry.enclosures:
if ('image' in enclosure.get('type')
and enclosure.get('href')):
img_tmp = NamedTemporaryFile(delete=True)
img_tmp.write(urlopen(enclosure['href']).read())
img_tmp.flush()
entry.image.save(os.path.basename(enclosure['href']),
File(img_tmp))
break
if self.default_author:
entry.authors.add(self.default_author)
elif feed_entry.get('author_detail'):
try:
author = Author.objects.create_user(
slugify(feed_entry.author_detail.get('name')),
feed_entry.author_detail.get('email', ''))
except IntegrityError:
author = Author.objects.get(
username=slugify(feed_entry.author_detail.get('name')))
entry.authors.add(author)
self.write_out(self.style.ITEM('OK\n'))
def run_diarios(self):
http = urllib3.PoolManager()
def reset_id_model(model):
query = """SELECT setval(pg_get_serial_sequence('"%(app_model_name)s"','id'),
coalesce(max("id"), 1), max("id") IS NOT null)
FROM "%(app_model_name)s";
""" % {
'app_model_name': _get_registration_key(model)
}
with connection.cursor() as cursor:
cursor.execute(query)
# get all the rows as a list
rows = cursor.fetchall()
print(rows)
tipo = Tipolei.objects.get(id=5)
docs = Documento.objects.filter(assuntos__tipo=tipo, publicado=True)
docs = docs.order_by('data_lei')
tipo_de_diario = TipoDeDiario.objects.get(pk=1)
# DiarioOficial.objects.all().delete()
for doc in docs:
try:
diario = DiarioOficial.objects.get(id=doc.id)
except:
diario = DiarioOficial()
diario.id = doc.id
diario.tipo = tipo_de_diario
diario.descricao = doc.epigrafe
diario.edicao = doc.numero
diario.data = doc.data_lei
diario.save()
normas = NormaJuridica.objects.filter(
veiculo_publicacao=diario.edicao)
if normas.exists():
diario.normas.add(*list(normas))
if diario.arquivo:
print('PULANDO', diario.id, DiarioOficial._meta.object_name)
continue
sleep(1)
print('Fazendo', diario.id, DiarioOficial._meta.object_name)
url = ('http://168.228.184.68:8580/portal/'
'downloadFile.pdf?sv=2&id=%s') % diario.id
request = None
try:
request = http.request('GET', url, timeout=10.0, retries=False)
if not request:
continue
if request.status == 404:
print(diario.pk, "não possui arquivo...")
continue
if not request.data or len(request.data) == 0:
continue
temp = NamedTemporaryFile(delete=True)
temp.write(request.data)
temp.flush()
try:
name_file = '%s_diario_oficial_%s.pdf' % (diario.id,
diario.edicao)
diario.arquivo.save(name_file, File(temp), save=True)
except Exception as e:
print(e)
except:
print(diario.pk, "erro...")
reset_id_model(DiarioOficial)
print(doc.id, doc.numero)
def __buscar_datos(self,soup,url):
fotos = soup.find_all('img', attrs={'class': 'foto'})
for foto in fotos:
numinv = foto['title'].split()[-1]
if numinv not in registros:
registros.append(numinv)
codigo = self.__obtener_codigo(numinv)
titulo = "(?)"
dimensiones = (0, 0)
fechaingreso = None
datacion = ""
descripcion = ""
observaciones = ""
soporte = ""
iconografias = []
tecnicas = []
all_row = soup.find_all('th', attrs={'class': 'tabla1TituloMB'})
for row in all_row:
if 'Título' in row.text.encode('utf-8').strip():
if numinv in row.parent.td['headers'][0]:
titulo = row.parent.td.text.encode('utf-8').strip()
if 'Dimensiones' in row.text.encode('utf-8').strip():
if numinv in row.parent.td['headers'][0]:
aux = row.parent.td.text.encode('utf-8').strip()
dimensiones = self.__obtener_dimensiones(aux)
if 'Datación' in row.text.encode('utf-8').strip():
if numinv in row.parent.td['headers'][0]:
datacion = row.parent.td.text.encode('utf-8').strip()
datacion = re.sub('[(]{1}.*[)]{1}','',datacion).strip()
if 'Descripción' in row.text.encode('utf-8').strip():
if numinv in row.parent.td['headers'][0]:
descripcion = row.parent.td.text.encode('utf-8').strip()
if 'Observaciones' in row.text.encode('utf-8').strip():
if numinv in row.parent.td['headers'][0]:
observaciones = row.parent.td.text.encode('utf-8').strip()
if 'Materia/Soporte' in row.text.encode('utf-8').strip():
if numinv in row.parent.td['headers'][0]:
soporte = row.parent.td.text.encode('utf-8').strip()
soporte = re.sub('[\[]{1}.*[\]]{1}','',soporte).strip()
if 'Iconografia' in row.text.encode('utf-8').strip():
if numinv in row.parent.td['headers'][0]:
iconografia = row.parent.td.text.encode('utf-8').strip()
iconografias = iconografia.replace(' ',';').split(';')
if 'Técnica' in row.text.encode('utf-8').strip():
if numinv in row.parent.td['headers'][0]:
tecnica = row.parent.td.text.encode('utf-8').strip()
tecnicas = tecnica.replace(' ',';').split(';')
img_url = "http://ceres.mcu.es/pages/Viewer?accion=42&Museo=&AMuseo=MCA&Ninv="+numinv
if debug:
print '\n\033[1m\033[33m' + codigo[0] + codigo[1] + "\033[0m"
print 'Título: ' + fix_string(titulo)
print 'Altura: ' + str(dimensiones[0]) + ' cm'
print 'Anchura: ' + str(dimensiones[1]) + ' cm'
print 'Datación: ' + fix_string(datacion)
print 'Descripción: ' + fix_string(descripcion[:75]) + "..."
print 'Observaciones: ' + fix_string(observaciones[:75]) + "..."
print 'Soporte: ' + fix_string(soporte)
if iconografias:
print 'Iconografías: '
for iconografia in iconografias:
print fix_string('\t'+iconografia[:30].strip())
if tecnicas:
print 'Técnicas: '
for tecnica in tecnicas:
print fix_string('\t'+re.sub('[\[]{1}.*[\]]{1}','',tecnica)[:30].strip())
print 'Imagen: ' + img_url
else:
try:
print "\033[33m[ ]\033[0m " + codigo[0] + codigo[1] + "\r",
sys.stdout.flush()
s = Soporte.objects.filter(nombre=fix_string(soporte))
if len(s):
objSoporte = s[0]
else:
if len(soporte) and len(soporte) <= 30:
objSoporte = Soporte(nombre=fix_string(soporte))
objSoporte.save()
else:
objSoporte = None
setIconografias = []
for iconografia in iconografias:
if len(iconografia) and len(iconografia) <= 30:
i = Iconografia.objects.filter(nombre=fix_string(iconografia.strip()))
if len(i):
objIconografia = i[0]
else:
objIconografia = Iconografia(nombre=fix_string(iconografia.strip()))
objIconografia.save()
setIconografias.append(objIconografia)
setTecnicas = []
for tecnica in tecnicas:
tecnica = re.sub('[\[]{1}.*[\]]{1}','',tecnica)
if len(tecnica) and len(tecnica) <= 30:
t = Tecnica.objects.filter(nombre=fix_string(tecnica.strip()))
if len(t):
objTecnica = t[0]
else:
objTecnica = Tecnica(nombre=fix_string(tecnica.strip()))
objTecnica.save()
setTecnicas.append(objTecnica)
arq = Arqueologia.objects.filter(numinv=codigo[1])
if len(arq):
arq[0].delete()
ba = Bellasartes(numinv=codigo[1])
ba.save_base()
ba.titulo = fix_string(titulo).split(";")[0][:100]
ba.codigo = fix_string(codigo[0])
ba.altura = dimensiones[0]
ba.ancho = dimensiones[1]
ba.datacion = fix_string(datacion).replace('=','-')
ba.descripcion = fix_string(descripcion)
ba.observaciones = fix_string(observaciones)
ba.fechaingreso = fechaingreso
ba.iconografia.set(setIconografias)
ba.tecnica.set(setTecnicas)
try:
ba.soporte = objSoporte
except:
pass
try:
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(urllib2.urlopen(img_url).read())
img_temp.flush()
ba.anverso.save(codigo[0]+'.jpg', File(img_temp))
except:
pass
ba.save()
print "\033[32m[+]\033[0m"
except:
print "\033[31m[x]\033[0m"
def test_to_xls_export_works(self):
survey = self._create_childrens_survey()
export_builder = ExportBuilder()
export_builder.set_survey(survey)
xls_file = NamedTemporaryFile(suffix='.xls')
filename = xls_file.name
export_builder.to_xls_export(filename, self.data)
xls_file.seek(0)
wb = load_workbook(filename)
# check that we have childrens_survey, children, children_cartoons
# and children_cartoons_characters sheets
expected_sheet_names = ['childrens_survey', 'children',
'children_cartoons',
'children_cartoons_characters']
self.assertEqual(wb.get_sheet_names(), expected_sheet_names)
# check header columns
main_sheet = wb.get_sheet_by_name('childrens_survey')
expected_column_headers = [
u'name', u'age', u'geo/geolocation', u'geo/_geolocation_latitude',
u'geo/_geolocation_longitude', u'geo/_geolocation_altitude',
u'geo/_geolocation_precision', u'tel/tel.office',
u'tel/tel.mobile', u'_id', u'meta/instanceID', u'_uuid',
u'_submission_time', u'_index', u'_parent_index',
u'_parent_table_name', u'_tags', '_notes']
column_headers = [c[0].value for c in main_sheet.columns]
self.assertEqual(sorted(column_headers),
sorted(expected_column_headers))
childrens_sheet = wb.get_sheet_by_name('children')
expected_column_headers = [
u'children/name', u'children/age', u'children/fav_colors',
u'children/fav_colors/red', u'children/fav_colors/blue',
u'children/fav_colors/pink', u'children/ice.creams',
u'children/ice.creams/vanilla', u'children/ice.creams/strawberry',
u'children/ice.creams/chocolate', u'_id', u'_uuid',
u'_submission_time', u'_index', u'_parent_index',
u'_parent_table_name', u'_tags', '_notes']
column_headers = [c[0].value for c in childrens_sheet.columns]
self.assertEqual(sorted(column_headers),
sorted(expected_column_headers))
cartoons_sheet = wb.get_sheet_by_name('children_cartoons')
expected_column_headers = [
u'children/cartoons/name', u'children/cartoons/why', u'_id',
u'_uuid', u'_submission_time', u'_index', u'_parent_index',
u'_parent_table_name', u'_tags', '_notes']
column_headers = [c[0].value for c in cartoons_sheet.columns]
self.assertEqual(sorted(column_headers),
sorted(expected_column_headers))
characters_sheet = wb.get_sheet_by_name('children_cartoons_characters')
expected_column_headers = [
u'children/cartoons/characters/name',
u'children/cartoons/characters/good_or_evil', u'_id', u'_uuid',
u'_submission_time', u'_index', u'_parent_index',
u'_parent_table_name', u'_tags', '_notes']
column_headers = [c[0].value for c in characters_sheet.columns]
self.assertEqual(sorted(column_headers),
sorted(expected_column_headers))
xls_file.close()
def post(self, request, *args, **kwargs):
solution = get_object_or_404(Solution, pk=kwargs.pop('pk'))
solver = Solver()
status = solver.status(solution.submission_id)
if status == Solver.SUCCESS:
info = solver.info(solution.submission_id)
solution.objects_in_field = ', '.join(info['objects_in_field'])
solution.ra = "%.3f" % info['calibration']['ra']
solution.dec = "%.3f" % info['calibration']['dec']
solution.orientation = "%.3f" % info['calibration']['orientation']
solution.radius = "%.3f" % info['calibration']['radius']
# Get the images 'w' and adjust pixscale
if solution.content_object:
w = solution.content_object.w
pixscale = info['calibration']['pixscale']
if w and pixscale:
hd_w = settings.THUMBNAIL_ALIASES['']['hd']['size'][0]
if hd_w > w:
hd_w = w
ratio = hd_w / float(w)
corrected_scale = float(pixscale) * ratio
solution.pixscale = "%.3f" % corrected_scale
else:
solution.pixscale = None
try:
target = solution.content_type.get_object_for_this_type(
pk=solution.object_id)
except solution.content_type.model_class().DoesNotExist:
# Target image was deleted meanwhile
context = {'status': Solver.FAILED}
return HttpResponse(simplejson.dumps(context),
content_type='application/json')
# Annotate image
annotations_obj = solver.annotations(solution.submission_id)
solution.annotations = simplejson.dumps(annotations_obj)
annotator = Annotator(solution)
annotated_image = annotator.annotate()
filename, ext = os.path.splitext(target.image_file.name)
annotated_filename = "%s-%d%s" % (filename, int(time.time()), ext)
if annotated_image:
solution.image_file.save(annotated_filename, annotated_image)
# Get sky plot image
url = solver.sky_plot_zoom1_image_url(solution.submission_id)
if url:
img = NamedTemporaryFile(delete=True)
img.write(urllib2.urlopen(url).read())
img.flush()
img.seek(0)
f = File(img)
try:
solution.skyplot_zoom1.save(target.image_file.name, f)
except IntegrityError:
pass
solution.status = status
solution.save()
context = {'status': solution.status}
return HttpResponse(simplejson.dumps(context),
content_type='application/json')
def test_zipped_csv_export_works(self):
survey = self._create_childrens_survey()
export_builder = ExportBuilder()
export_builder.set_survey(survey)
temp_zip_file = NamedTemporaryFile(suffix='.zip')
export_builder.to_zipped_csv(temp_zip_file.name, self.data)
temp_zip_file.seek(0)
temp_dir = tempfile.mkdtemp()
zip_file = zipfile.ZipFile(temp_zip_file.name, "r")
zip_file.extractall(temp_dir)
zip_file.close()
temp_zip_file.close()
# generate data to compare with
index = 1
indices = {}
survey_name = survey.name
outputs = []
for d in self.data:
outputs.append(
dict_to_joined_export(d, index, indices, survey_name))
index += 1
# check that each file exists
self.assertTrue(
os.path.exists(
os.path.join(temp_dir, "{0}.csv".format(survey.name))))
with open(
os.path.join(
temp_dir, "{0}.csv".format(survey.name))) as csv_file:
reader = csv.reader(csv_file)
rows = [r for r in reader]
# open comparison file
with open(_logger_fixture_path(
'csvs', 'childrens_survey.csv')) as fixture_csv:
fixture_reader = csv.reader(fixture_csv)
expected_rows = [r for r in fixture_reader]
self.assertEqual(rows, expected_rows)
self.assertTrue(
os.path.exists(
os.path.join(temp_dir, "children.csv")))
with open(os.path.join(temp_dir, "children.csv")) as csv_file:
reader = csv.reader(csv_file)
rows = [r for r in reader]
# open comparison file
with open(_logger_fixture_path(
'csvs', 'children.csv')) as fixture_csv:
fixture_reader = csv.reader(fixture_csv)
expected_rows = [r for r in fixture_reader]
self.assertEqual(rows, expected_rows)
self.assertTrue(
os.path.exists(
os.path.join(temp_dir, "children_cartoons.csv")))
with open(os.path.join(temp_dir, "children_cartoons.csv")) as csv_file:
reader = csv.reader(csv_file)
rows = [r for r in reader]
# open comparison file
with open(_logger_fixture_path(
'csvs', 'children_cartoons.csv')) as fixture_csv:
fixture_reader = csv.reader(fixture_csv)
expected_rows = [r for r in fixture_reader]
self.assertEqual(rows, expected_rows)
self.assertTrue(
os.path.exists(
os.path.join(temp_dir, "children_cartoons_characters.csv")))
with open(os.path.join(
temp_dir, "children_cartoons_characters.csv")) as csv_file:
reader = csv.reader(csv_file)
rows = [r for r in reader]
# open comparison file
with open(_logger_fixture_path(
'csvs',
'children_cartoons_characters.csv')) as fixture_csv:
fixture_reader = csv.reader(fixture_csv)
expected_rows = [r for r in fixture_reader]
self.assertEqual(rows, expected_rows)
shutil.rmtree(temp_dir)
def test_partial_iati_import(self):
"""
Test an IATI import that ignores certain elements
Ignore the data if the akvo:import attribute set to falsey values, i.e. "false", "no", "f"
or "0". Elements that support this are: budget, contact-info, humanitarian-scope,
legacy-data, location, participating-org, planned-disbursement, policy-marker,
recipient-country, recipient-region, related-activity, result, sector and transaction
"""
#import a project
iati_v2_import = IatiImport.objects.create(label="Test IATI v2 import",
user=self.user)
iati_v2_xml_file = NamedTemporaryFile(delete=True)
iati_v2_xml_file.write(IATI_V2_STRING)
iati_v2_xml_file.flush()
iati_v2_import_job = IatiImportJob.objects.create(
iati_import=iati_v2_import, iati_xml_file=File(iati_v2_xml_file))
iati_v2_import_job.run()
project_v2 = Project.objects.get(
iati_activity_id="NL-KVK-0987654321-v2")
self.assertEqual(project_v2.contacts.count(), 1)
contact_info = project_v2.contacts.all()[0]
self.assertEqual(contact_info.organisation, "Agency A")
self.assertEqual(contact_info.department, "Department B")
self.assertEqual(project_v2.locations.count(), 2)
location_1 = project_v2.locations.get(reference="AF-KAN")
location_2 = project_v2.locations.get(reference="KH-PNH")
self.assertEqual(location_1.location_code, "1453782")
self.assertEqual(location_2.location_code, "1821306")
# do a new import to the same project, with contact and location elements having
# akvo:import="false"
partial_import = IatiImport.objects.create(
label="Test partial IATI import", user=self.user)
partial_import_xml_file = NamedTemporaryFile(delete=True)
partial_import_xml_file.write(IATI_PARTIAL_IMPORT)
partial_import_xml_file.flush()
partial_import_job = IatiImportJob.objects.create(
iati_import=partial_import,
iati_xml_file=File(partial_import_xml_file))
partial_import_job.run()
project_partial_import = Project.objects.get(
iati_activity_id="NL-KVK-0987654321-v2")
self.assertIsInstance(project_partial_import, Project)
# Assert that no data has changed, even if the XML has
self.assertEqual(project_v2.results.count(), 1)
result_1 = project_v2.results.get(title="Result title")
self.assertEqual(result_1.indicators.count(), 1)
self.assertEqual(
result_1.indicators.all()[0].periods.all()[0].actual_value, u'11')
self.assertEqual(project_v2.contacts.count(), 1)
contact_info = project_v2.contacts.all()[0]
self.assertEqual(contact_info.organisation, "Agency A")
self.assertEqual(contact_info.department, "Department B")
self.assertEqual(project_v2.locations.count(), 2)
location_1 = project_v2.locations.get(reference="AF-KAN")
location_2 = project_v2.locations.get(reference="KH-PNH")
self.assertEqual(location_1.location_code, "1453782")
self.assertEqual(location_2.location_code, "1821306")
self.assertEqual(project_v2.humanitarian_scopes.count(), 2)
humanitarian_scope_1 = project_v2.humanitarian_scopes.get(
vocabulary="1-2")
humanitarian_scope_2 = project_v2.humanitarian_scopes.get(
vocabulary="99")
self.assertEqual(humanitarian_scope_1.code, "2015-000050")
self.assertEqual(humanitarian_scope_2.vocabulary_uri,
"http://example.com/vocab.html")
# three participating orgs, and one reporting org
self.assertEqual(project_v2.partners.count(), 4)
self.assertEqual(project_v2.planned_disbursements.count(), 2)
self.assertEqual(project_v2.policy_markers.count(), 3)
self.assertEqual(project_v2.recipient_countries.count(), 2)
self.assertEqual(project_v2.recipient_regions.count(), 3)
related_project_1 = project_v2.related_projects.get(
related_iati_id="AA-AAA-123456789-6789")
self.assertIsInstance(related_project_1, RelatedProject)
self.assertEqual(project_v2.sectors.count(), 3)
self.assertEqual(project_v2.transactions.count(), 1)
self.assertEqual(project_v2.budget_items.count(), 1)
def handle(self, **options):
if not settings.MEDIA_ROOT:
raise ImproperlyConfigured(
'Please set MEDIA_ROOT in your settings file')
remote_url = options['remote_url']
try:
val = URLValidator()
val(remote_url)
except ValidationError:
raise CommandError('Please enter a valid URL')
exercise_api = "{0}/api/v2/exercise/?limit=999"
image_api = "{0}/api/v2/exerciseimage/?exercise={1}"
thumbnail_api = "{0}/api/v2/exerciseimage/{1}/thumbnails/"
headers = {
'User-agent':
default_user_agent('wger/{} + requests'.format(get_version()))
}
# Get all exercises
result = requests.get(exercise_api.format(remote_url),
headers=headers).json()
for exercise_json in result['results']:
exercise_name = exercise_json['name'].encode('utf-8')
exercise_uuid = exercise_json['uuid']
exercise_id = exercise_json['id']
self.stdout.write('')
self.stdout.write(
u"*** Processing {0} (ID: {1}, UUID: {2})".format(
exercise_name, exercise_id, exercise_uuid))
try:
exercise = Exercise.objects.get(uuid=exercise_uuid)
except Exercise.DoesNotExist:
self.stdout.write(
' Remote exercise not found in local DB, skipping...')
continue
# Get all images
images = requests.get(image_api.format(remote_url, exercise_id),
headers=headers).json()
if images['count']:
for image_json in images['results']:
image_id = image_json['id']
result = requests.get(thumbnail_api.format(
remote_url, image_id),
headers=headers).json()
image_name = os.path.basename(result['original'])
self.stdout.write(' Fetching image {0} - {1}'.format(
image_id, image_name))
try:
image = ExerciseImage.objects.get(pk=image_id)
self.stdout.write(
' --> Image already present locally, skipping...'
)
continue
except ExerciseImage.DoesNotExist:
self.stdout.write(
' --> Image not found in local DB, creating now...'
)
image = ExerciseImage()
image.pk = image_id
# Save the downloaded image, see link for details
# http://stackoverflow.com/questions/1308386/programmatically-saving-image-to-
retrieved_image = requests.get(result['original'],
headers=headers)
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(retrieved_image.content)
img_temp.flush()
image.exercise = exercise
image.is_main = image_json['is_main']
image.status = image_json['status']
image.image.save(
os.path.basename(image_name),
File(img_temp),
)
image.save()
else:
self.stdout.write(
' No images for this exercise, nothing to do')
def to_zipped_csv(self, path, data, *args):
def write_row(row, csv_writer, fields):
csv_writer.writerow(
[encode_if_str(row, field) for field in fields])
csv_defs = {}
for section in self.sections:
csv_file = NamedTemporaryFile(suffix=".csv")
csv_writer = csv.writer(csv_file)
csv_defs[section['name']] = {
'csv_file': csv_file,
'csv_writer': csv_writer
}
# write headers
for section in self.sections:
fields = [element['title'] for element in section['elements']]\
+ self.EXTRA_FIELDS
csv_defs[section['name']]['csv_writer'].writerow(
[f.encode('utf-8') for f in fields])
index = 1
indices = {}
survey_name = self.survey.name
for d in data:
# decode mongo section names
joined_export = dict_to_joined_export(d, index, indices,
survey_name)
output = ExportBuilder.decode_mongo_encoded_section_names(
joined_export)
# attach meta fields (index, parent_index, parent_table)
# output has keys for every section
if survey_name not in output:
output[survey_name] = {}
output[survey_name][INDEX] = index
output[survey_name][PARENT_INDEX] = -1
for section in self.sections:
# get data for this section and write to csv
section_name = section['name']
csv_def = csv_defs[section_name]
fields = [element['xpath'] for element in section['elements']
] + self.EXTRA_FIELDS
csv_writer = csv_def['csv_writer']
# section name might not exist within the output, e.g. data was
# not provided for said repeat - write test to check this
row = output.get(section_name, None)
if type(row) == dict:
write_row(self.pre_process_row(row, section), csv_writer,
fields)
elif type(row) == list:
for child_row in row:
write_row(self.pre_process_row(child_row, section),
csv_writer, fields)
index += 1
# write zipfile
with ZipFile(path, 'w') as zip_file:
for section_name, csv_def in csv_defs.iteritems():
csv_file = csv_def['csv_file']
csv_file.seek(0)
zip_file.write(csv_file.name,
"_".join(section_name.split("/")) + ".csv")
# close files when we are done
for section_name, csv_def in csv_defs.iteritems():
csv_def['csv_file'].close()
def parse(headers, url) -> (Product, ProductPage):
resp = requests.get(url, headers=headers)
content = str(resp.content)
parser = DetailParser(content)
detail = parser.parse()
product_title = detail.get('title')
product_reviews_number = detail.get('reviews')
price_elements = parser.selector.css('#priceblock_saleprice') or parser.selector.css('.offer-price')
product_price = None
if len(price_elements) > 0:
product_price = price_elements[0].root.text
product = Product(
product_title=product_title.strip().replace('\n', '').replace('\\n', '').replace(' ', ''),
url=url,
price=float(product_price.replace('$', '')),
rating_amount=float(product_reviews_number or 0)
)
def get_number(str_with_num) -> List[float]:
return [item[1] if item[1] else item[0] for item in re.findall('(\d+\.\d+)|(\d+)', value)]
def get_first_float(numbers: List[float]) -> float:
return float(str(numbers[0])) if len(numbers) else None
for key, value in detail['details'].items():
if 'Shipping Weight' in key:
numbers = get_number(value)
product.shipping_weight = get_first_float(numbers)
elif 'Weight' in key:
numbers = get_number(value)
product.weight = get_first_float(numbers)
elif 'Dimensions' in key:
numbers = get_number(value)
product.dim_x, product.dim_y, product.dim_z = numbers
elif 'model number' in key:
product.model_number = value.replace(' ', '').replace('\n', '').replace('\\n', '')
elif 'ASIN' in key:
product.asin = value.replace(' ', '').replace('\n', '').replace('\\n', '')
elif 'Best Sellers Rank' in key:
numbers = get_number(value)
product.bsr = get_first_float(numbers)
content = ProductPage(page_html=content)
images = []
try:
image_responses = parser.get_image_responses()
for url, response in image_responses:
hash_sum = md5()
image_temp_file = NamedTemporaryFile(delete=True)
for chunk in response.iter_content(1024):
if not chunk:
break
hash_sum.update(chunk)
image_temp_file.write(chunk)
dig = hash_sum.hexdigest()
image_temp_file.flush()
filename = f'{product.asin}_{dig}'
temp_file = File(image_temp_file, name=filename)
image = ProductImage(
name=filename,
hash=dig,
product=product,
)
image.file.save(filename, temp_file)
images.append(image)
except:
pass
return product, content, images
def to_zipped_sav(self, path, data, *args):
def write_row(row, csv_writer, fields):
sav_writer.writerow(
[encode_if_str(row, field, True) for field in fields])
sav_defs = {}
# write headers
for section in self.sections:
fields = [element['title'] for element in section['elements']]\
+ self.EXTRA_FIELDS
c = 0
var_labels = {}
var_names = []
tmp_k = {}
for field in fields:
c += 1
var_name = 'var%d' % c
var_labels[var_name] = field
var_names.append(var_name)
tmp_k[field] = var_name
var_types = dict(
[(tmp_k[element['title']],
0 if element['type'] in ['decimal', 'int'] else 255)
for element in section['elements']] +
[(tmp_k[item],
0 if item in ['_id', '_index', '_parent_index'] else 255)
for item in self.EXTRA_FIELDS])
sav_file = NamedTemporaryFile(suffix=".sav")
sav_writer = SavWriter(sav_file.name,
varNames=var_names,
varTypes=var_types,
varLabels=var_labels,
ioUtf8=True)
sav_defs[section['name']] = {
'sav_file': sav_file,
'sav_writer': sav_writer
}
index = 1
indices = {}
survey_name = self.survey.name
for d in data:
# decode mongo section names
joined_export = dict_to_joined_export(d, index, indices,
survey_name)
output = ExportBuilder.decode_mongo_encoded_section_names(
joined_export)
# attach meta fields (index, parent_index, parent_table)
# output has keys for every section
if survey_name not in output:
output[survey_name] = {}
output[survey_name][INDEX] = index
output[survey_name][PARENT_INDEX] = -1
for section in self.sections:
# get data for this section and write to csv
section_name = section['name']
sav_def = sav_defs[section_name]
fields = [element['xpath'] for element in section['elements']
] + self.EXTRA_FIELDS
sav_writer = sav_def['sav_writer']
row = output.get(section_name, None)
if type(row) == dict:
write_row(self.pre_process_row(row, section), sav_writer,
fields)
elif type(row) == list:
for child_row in row:
write_row(self.pre_process_row(child_row, section),
sav_writer, fields)
index += 1
for section_name, sav_def in sav_defs.iteritems():
sav_def['sav_writer'].closeSavFile(sav_def['sav_writer'].fh,
mode='wb')
# write zipfile
with ZipFile(path, 'w') as zip_file:
for section_name, sav_def in sav_defs.iteritems():
sav_file = sav_def['sav_file']
sav_file.seek(0)
zip_file.write(sav_file.name,
"_".join(section_name.split("/")) + ".sav")
# close files when we are done
for section_name, sav_def in sav_defs.iteritems():
sav_def['sav_file'].close()
def classify_api(request):
data = {"success": False}
clean_directory()
if request.method == "POST":
model = request.POST.get("model", None)
if model == 'imagenet':
tmp_f = NamedTemporaryFile()
tmp_adver = NamedTemporaryFile()
tmp_adver_gif = NamedTemporaryFile()
if request.FILES.get("image", None) is not None:
image_request = request.FILES["image"]
image_bytes = image_request.read()
image.save(tmp_f, image.format)
elif request.POST.get("image64", None) is not None:
base64_data = request.POST.get("image64", None).split(',',
1)[1]
plain_data = base64.b64decode(base64_data)
image = Image.open(io.BytesIO(plain_data))
image.save(
os.path.join(current_dir,
'imagenet/dataset/images/testtest.png'))
tmp_f.write(plain_data)
tmp_f.close()
# Backend session for attack
print('Building Backend Session.')
K.set_learning_phase(0)
sess = tf.Session()
backend.set_session(sess)
# Image preprocess
print('Modifying image')
x = np.expand_dims(preprocess(image.resize((299, 299))), axis=0)
img_shape = [1, 299, 299, 3]
x_input = tf.placeholder(tf.float32, shape=img_shape)
# Define model
d = discriminator()
# Prediction of original image
print('prediction of original image')
classify_result = get_predictions(d, x, 10)
# Select attack algorithm and iteration
attack_algorithm = request.POST.get("attack", None)
n = int(request.POST.get("iterate", None))
# Start attack
result, attack_speed = attack(attack_algorithm, n, d, x_input, x,
sess)
print("attack speed: %s seconds" % (round(attack_speed, 5)))
print('original image:', classify_result[0][1])
print('adversarial example is classified by', result[0][1])
# Print image to web site
with open(
os.path.join(current_dir, 'imagenet/output/testtest.png'),
'rb') as img_file:
img_str = base64.b64encode(img_file.read())
tmp_adver.write(base64.b64decode(img_str))
tmp_adver.close()
with open(
os.path.join(current_dir, 'imagenet/output/testtest.gif'),
'rb') as gif_file:
gif_str = base64.b64encode(gif_file.read())
tmp_adver_gif.write(base64.b64decode(gif_str))
tmp_adver_gif.close()
elif model == 'mnist':
tmp_adver = NamedTemporaryFile()
tmp_adver_gif = NamedTemporaryFile()
tmp_f = NamedTemporaryFile()
mnist_sample = int(request.POST.get("sample", None))
mnist_target = int(request.POST.get("target", None))
mnist_algorithm = request.POST.get("mnist_algorithm", None)
sample_result, adver_result, attack_speed = mnist_attack_func(
mnist_sample, mnist_target, mnist_algorithm)
print("attack speed: %s seconds" % (round(attack_speed, 5)))
print('original class:', mnist_sample, 'target class:',
mnist_target)
print('adversarial example is classified by',
np.argmax(adver_result))
sample_result = sample_result.tolist()
adver_result = adver_result.tolist()
with open(
os.path.join(current_dir,
'mnist/dataset/images/testtest.png'),
'rb') as input_file:
input_str = base64.b64encode(input_file.read())
tmp_f.write(base64.b64decode(input_str))
tmp_f.close()
with open(os.path.join(current_dir, 'mnist/output/testtest.png'),
'rb') as img_file:
img_str = base64.b64encode(img_file.read())
tmp_adver.write(base64.b64decode(img_str))
tmp_adver.close()
with open(os.path.join(current_dir, 'mnist/output/testtest.gif'),
'rb') as gif_file:
gif_str = base64.b64encode(gif_file.read())
tmp_adver_gif.write(base64.b64decode(gif_str))
tmp_adver_gif.close()
# Make Graph
data["attack_speed"] = attack_speed
data["success"] = True
data["confidence"] = {}
if model == 'imagenet':
data["model"] = 'imagenet'
for i in range(len(classify_result)):
data["confidence"][classify_result[i][1]] = float(
classify_result[i][2])
data["adverimage"] = 'data:image/png;base64,' + img_str.decode(
'utf-8')
data["adverimage_gif"] = 'data:image/gif;base64,' + gif_str.decode(
'utf-8')
data["adversarial"] = {}
for i in range(len(result)):
data["adversarial"][result[i][1]] = float(result[i][2])
sess.close()
elif model == 'mnist':
data["model"] = 'mnist'
for i in range(len(sample_result[0])):
data["confidence"][str(i)] = float(sample_result[0][i])
data["input_image"] = 'data:image/png;base64,' + input_str.decode(
'utf-8')
data["adverimage"] = 'data:image/png;base64,' + img_str.decode(
'utf-8')
data["adverimage_gif"] = 'data:image/gif;base64,' + gif_str.decode(
'utf-8')
data["adversarial"] = {}
for i in range(len(adver_result[0])):
data["adversarial"][str(i)] = float(adver_result[0][i])
return JsonResponse(data)
def restore_documents(
restore_file: io.BytesIO,
user: User,
skip_ocr=False
):
restore_file.seek(0)
with tarfile.open(fileobj=restore_file, mode="r") as restore_archive:
backup_json = restore_archive.extractfile('backup.json')
backup_info = json.load(backup_json)
leading_user_in_path = False
_user = user
if not user:
leading_user_in_path = True
# user was not specified. It is assument that
# backup.json contains a list of users.
# Thus recreate users first.
for backup_user in backup_info['users']:
User.objects.create(
username=backup_user['username'],
email=backup_user['email'],
is_active=backup_user['is_active'],
is_superuser=backup_user['is_superuser']
)
for restore_file in restore_archive.getnames():
if restore_file == "backup.json":
continue
splitted_path = PurePath(restore_file).parts
base, ext = os.path.splitext(splitted_path[-1])
# if there is leading username, remove it.
if leading_user_in_path:
username = splitted_path[0]
_user = User.objects.get(username=username)
splitted_path = splitted_path[1:]
if backup_info.get('documents', False):
backup_info_documents = backup_info['documents']
else:
backup_info_documents = _get_json_user_documents_list(
backup_info,
_user
)
leading_user_in_path = True
for info in backup_info_documents:
document_info = info
if info['path'] == restore_file:
break
parent = None
# we first have to create a folder structure
if len(splitted_path) > 1:
for folder in splitted_path[:-1]:
folder_object = Folder.objects.filter(
title=folder,
user=_user
).filter(parent=parent).first()
if folder_object is None:
new_folder = Folder.objects.create(
title=folder,
parent=parent,
user=_user
)
parent = new_folder
else:
parent = folder_object
document_object = Document.objects.filter(
title=splitted_path[-1], user=_user
).filter(parent=parent).first()
if document_object is not None:
logger.error(
"Document %s already exists, skipping", restore_file
)
else:
with NamedTemporaryFile("w+b", suffix=ext) as temp_output:
temp_output.write(
restore_archive.extractfile(restore_file).read()
)
temp_output.seek(0)
size = os.path.getsize(temp_output.name)
page_count = get_pagecount(temp_output.name)
if parent:
parent_id = parent.id
else:
parent_id = None
new_doc = Document.create_document(
user=_user,
title=splitted_path[-1],
size=size,
lang=document_info['lang'],
file_name=splitted_path[-1],
parent_id=parent_id,
notes="",
page_count=page_count)
default_storage.copy_doc(
src=temp_output.name,
dst=new_doc.path.url()
)
for page_num in range(1, page_count + 1):
if not skip_ocr:
ocr_page.apply_async(kwargs={
'user_id': _user.id,
'document_id': new_doc.id,
'file_name': splitted_path[-1],
'page_num': page_num,
'lang': document_info['lang']}
)
def download(self, request):
"""
Download individual or multiple media assets. Multiple media assets are
zipped.
"""
# get pks
pks = get_pks(request.GET)
# get list of media
media = Media.objects.filter(pk__in=pks).order_by('caption')
n_media = media.count()
# no media?
if n_media == 0:
raise Http404(
'Unable to export media assets for empty list of media objects.'
)
# single media item?
if n_media == 1:
item = media.first()
response = FileResponse(open(item.original_path, 'rb'))
response[
'Content-Disposition'] = 'attachment; filename="%s"' % item.filename
return response
# multiple assets -> create zip file
f = NamedTemporaryFile()
zf = zipfile.ZipFile(f, 'w')
# attach original files to zip, handle duplicated filenames
filenames = {}
for item in media:
# determine unique filename
filename = item.filename
if filename not in filenames:
filenames[filename] = 1
else:
fn = filename
base, ext = os.path.splitext(filename)
while fn in filenames:
filenames[fn] += 1
fn = '%s-%d%s' % (base, filenames[fn], ext)
filename = fn
filenames[filename] = 1
# attach file to zip...
zf.write(item.original_path, filename)
zf.close()
f.seek(0)
# determine site name from settings (CMS)
if 'cubane.cms' in settings.INSTALLED_APPS:
from cubane.cms.views import get_cms_settings
site_name = get_cms_settings().name
if site_name is None: site_name = ''
site_name = slugify(site_name)
if site_name != '': site_name += '_'
else:
site_name = ''
# generate download filename for zip file
today = datetime.date.today()
filename = '{0}media_{1:02d}_{2:02d}_{3:04d}.zip'.format(
site_name, today.day, today.month, today.year)
# serve file
response = FileResponse(f)
response[
'Content-Disposition'] = 'attachment; filename="%s"' % filename
return response
def input(self, **kwargs):
encoding = self.default_encoding
options = dict(self.options)
if self.infile is None and "{infile}" in self.command:
# create temporary input file if needed
if self.filename is None:
self.infile = NamedTemporaryFile(mode='wb')
self.infile.write(self.content.encode(encoding))
self.infile.flush()
options["infile"] = self.infile.name
else:
# we use source file directly, which may be encoded using
# something different than utf8. If that's the case file will
# be included with charset="something" html attribute and
# charset will be available as filter's charset attribute
encoding = self.charset # or self.default_encoding
self.infile = open(self.filename)
options["infile"] = self.filename
if "{outfile}" in self.command and "outfile" not in options:
# create temporary output file if needed
ext = self.type and ".%s" % self.type or ""
self.outfile = NamedTemporaryFile(mode='r+', suffix=ext)
options["outfile"] = self.outfile.name
# Quote infile and outfile for spaces etc.
if "infile" in options:
options["infile"] = shell_quote(options["infile"])
if "outfile" in options:
options["outfile"] = shell_quote(options["outfile"])
try:
command = self.command.format(**options)
proc = subprocess.Popen(command,
shell=True,
cwd=self.cwd,
stdout=self.stdout,
stdin=self.stdin,
stderr=self.stderr)
if self.infile is None:
# if infile is None then send content to process' stdin
filtered, err = proc.communicate(self.content.encode(encoding))
else:
filtered, err = proc.communicate()
filtered, err = filtered.decode(encoding), err.decode(encoding)
except (IOError, OSError) as e:
raise FilterError('Unable to apply %s (%r): %s' %
(self.__class__.__name__, self.command, e))
else:
if proc.wait() != 0:
# command failed, raise FilterError exception
if not err:
err = ('Unable to apply %s (%s)' %
(self.__class__.__name__, self.command))
if filtered:
err += '\n%s' % filtered
raise FilterError(err)
if self.verbose:
self.logger.debug(err)
outfile_path = options.get('outfile')
if outfile_path:
with io.open(outfile_path, 'r', encoding=encoding) as file:
filtered = file.read()
finally:
if self.infile is not None:
self.infile.close()
if self.outfile is not None:
self.outfile.close()
return smart_text(filtered)
def test_articolo(self):
CONTENUTO_1 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit.\n"
NOME_1 = 'Test1.txt'
CONTENUTO_2 = "Donec tempus nisi eu enim consequat, non scelerisque nisi accumsan.\n"
NOME_2 = 'Test/Test2.txt'
volontario, _, _ = crea_persona_sede_appartenenza()
presidente = crea_persona()
presidente.save()
presidente, sede, _ = crea_persona_sede_appartenenza(presidente)
delega_presidente_in_corso = Delega(
persona=presidente,
tipo=PRESIDENTE,
oggetto=sede,
inizio=datetime.datetime.now() - datetime.timedelta(days=5),
fine=datetime.datetime.now() + datetime.timedelta(days=5)
)
delega_presidente_in_corso.save()
articolo = Articolo.objects.create(
titolo='Titolo 1',
corpo=parola_casuale(3000),
data_inizio_pubblicazione=datetime.datetime.now() - datetime.timedelta(days=5),
)
self.assertEqual(articolo.corpo[:Articolo.DIMENSIONE_ESTRATTO], articolo.estratto)
self.assertFalse(articolo.termina)
articolo2 = Articolo.objects.create(
titolo='Titolo 2',
corpo='Testo random',
estratto='estratto qualsiasi',
data_inizio_pubblicazione=datetime.datetime.now() - datetime.timedelta(days=5),
data_fine_pubblicazione=datetime.datetime.now() + datetime.timedelta(days=5),
stato=Articolo.PUBBLICATO
)
segmento_presidenti_no_filtri = ArticoloSegmento.objects.create(
segmento='I',
articolo=articolo2,
)
self.assertNotEqual(articolo2.corpo, articolo2.estratto)
self.assertEqual(articolo2.estratto, 'estratto qualsiasi')
self.assertTrue(articolo2.termina)
articolo3 = Articolo.objects.create(
titolo='Titolo 3',
corpo='Testo qualsiasi',
estratto='estratto random',
data_inizio_pubblicazione=datetime.datetime.now() - datetime.timedelta(days=5),
stato=Articolo.PUBBLICATO
)
segmento_volontari_no_filtri = ArticoloSegmento.objects.create(
segmento='B',
articolo=articolo3
)
z = Zip(oggetto=articolo3)
f1 = NamedTemporaryFile(delete=False, mode='wt')
f1.write(CONTENUTO_1)
f1.close()
z.aggiungi_file(f1.name, NOME_1)
z.comprimi_e_salva(nome='TestZip.zip')
self.assertEqual(1, articolo3.allegati.all().count())
self.assertIn(z, articolo3.allegati.all())
articolo4 = Articolo.objects.create(
titolo='Titolo 4',
corpo='Testo qualsiasi 2',
estratto='estratto random 2',
data_inizio_pubblicazione=datetime.datetime.now() - datetime.timedelta(days=5),
data_fine_pubblicazione=datetime.datetime.now() - datetime.timedelta(days=2),
stato=Articolo.PUBBLICATO
)
pubblicati = Articolo.objects.pubblicati()
bozze = Articolo.objects.bozze()
self.assertEqual(pubblicati.count(), 2)
self.assertEqual(bozze.count(), 1)
self.assertIn(articolo, bozze)
self.assertNotIn(articolo, pubblicati)
self.assertNotIn(articolo2, bozze)
self.assertIn(articolo2, pubblicati)
self.assertNotIn(articolo3, bozze)
self.assertIn(articolo3, pubblicati)
self.assertNotIn(articolo4, bozze)
self.assertNotIn(articolo4, pubblicati)
segmenti_volontario = ArticoloSegmento.objects.all().filtra_per_segmenti(volontario)
articoli_volontario = segmenti_volontario.oggetti_collegati()
self.assertNotIn(articolo2, articoli_volontario)
self.assertIn(articolo3, articoli_volontario)
segmenti_presidente = ArticoloSegmento.objects.all().filtra_per_segmenti(presidente)
articoli_presidente = segmenti_presidente.oggetti_collegati()
self.assertIn(articolo2, articoli_presidente)
self.assertIn(articolo3, articoli_presidente)
print "total=%s, count=%s, exist=%s, new=%s" % (total, count, exist, new)
continue
else: #если отрицательно, создаем чек
print 'CREATE CHECK %s' % count, i['file']
new = new+1
c=check.objects.create(fname=i['file'])
print "total=%s, count=%s, exist=%s, new=%s" % (total, count, exist, new)
#
print 'READ FILE %s' % count, i['file']
#читаем файл, перекодируем в utf8
tmp=open('%s/%s' % (i['root'], i['file'])).read().decode('cp1251').encode('utf8')
#создаем временный файл
f = NamedTemporaryFile(delete=True)
f.write(tmp) #пишем открытый файл во временный
f.close()
#ПИШЕМ ЗАГОЛОВОК ЧЕКА, ТОЛЬКО ПЕРВУЮ СТРОКУ, ОСТАЛЬНЫЕ ДУБЛЬ
with open(f.name) as csvfile:
reader = csv.DictReader(csvfile, delimiter=';', quotechar='"')
for value in reader:
print "HEADER %s %s %s" % (value['НомерЧекаККМ'], value['Дата'], value['Цена'])
#берем дату
getdate=datetime.datetime.strptime(value['Дата'],"%d.%m.%Y %H:%M:%S")
#костыли
try: #пробуем взять магазины
def get_bootstrapdata(self):
url = 'https://www.airbnb.co.kr/s/homes?query=서울특별시§ion_offset=5'
headers = {
# 'cache-control': "no-cache",
# 'user-agent': 'Mozilla/5.0',
# 'postman-token': '912622a3-b1b0-49ab-89e9-ae6ab262184f',
# 'host': 'www.airbnb.co.kr',
# 'accept': '*/*',
# 'accept-encoding': 'gzip, deflate',
# 'cookie': '__svt=1381; cache_state=0; bev=1523934971_5ussJFyJsy%2BcIwMA; _csrf_token=V4%24.airbnb.co.kr%24G6i45M0XMrs%24reAsEi8IPWdU9YJsn2apAPDEIOtNCuY26kcvWPawU70%3D; jitney_client_session_id=2f0abc70-0be3-4f8d-aac3-84363faa4a73; jitney_client_session_created_at=1523957423; jitney_client_session_updated_at=1523957423; _user_attributes=%7B%22curr%22%3A%22KRW%22%2C%22guest_exchange%22%3A1072.3055650000001%2C%22device_profiling_session_id%22%3A%221523934971--2a4ffad382218d61b920a404%22%2C%22giftcard_profiling_session_id%22%3A%221523957424--c718f2fe50a984671b19354d%22%2C%22reservation_profiling_session_id%22%3A%221523957424--48f5ec024f0477b119d21220%22%7D; flags=268697600; dtc_exp=1; 9f2398a3e=control; b3b78300e=control; 65e98c419=control; a8f0b01c0=control; 4b522145b=treatment',
'cookie': '__svt=1381; cache_state=0; bev=1523934971_5ussJFyJsy%2BcIwMA; _csrf_token=V4%24.airbnb.co.kr%24G6i45M0XMrs%24reAsEi8IPWdU9YJsn2apAPDEIOtNCuY26kcvWPawU70%3D; jitney_client_session_id=7409e864-df24-46e1-9331-5793fc836ac4; jitney_client_session_created_at=1523979136; jitney_client_session_updated_at=1523979136; _user_attributes=%7B%22curr%22%3A%22KRW%22%2C%22guest_exchange%22%3A1072.3055650000001%2C%22device_profiling_session_id%22%3A%221523934971--2a4ffad382218d61b920a404%22%2C%22giftcard_profiling_session_id%22%3A%221523979138--ded77b2c438cd5e5b8f64f9e%22%2C%22reservation_profiling_session_id%22%3A%221523979138--403d68f09bc045f3db938d8a%22%7D; flags=268697600; dtc_exp=1; 9f2398a3e=control; b3b78300e=control; 65e98c419=control; a8f0b01c0=control; 4b522145b=treatment; Cookie_1=value; mdr_browser=desktop',
}
# 1) Airbnb web crawling
# response = requests.get(url)
# response = self.r.get(url, headers=headers)
# response = requests.get(url, headers=headers)
# print(response.status_code)
# source = response.text
# 2) local airbnb_data.html
source = open('airbnb_local_data.html', 'rt', encoding='utf8').read()
print(source)
bootstrap_data = re.search(r'data-hypernova-key="spaspabundlejs" data-hypernova-id=".*?"><!--(.*?)--></script>', source)
# print(bootstrap_data.groups(1)[0:10])
bootstrap_json = json.loads(bootstrap_data.group(1))
# listing 18개가 들어있는 list
listing_list = \
bootstrap_json['bootstrapData']['reduxData']['exploreTab']['response']['explore_tabs'][0]['sections'][0][
'listings']
for i in range(len(listing_list)):
print(listing_list[i]['listing'])
listing = listing_list[i]['listing']
# crawling data에서 image 다운받기
# img_cover_binary_data = requests.get(listing['picture_url']).content
response = requests.get(listing['picture_urls'][0]).content
print(type(response))
print(response)
print(ContentFile(response))
print(type(ContentFile(response)))
# print(ContentFile(requests.get(listing['picture_urls'][2]).content))
temp_file = NamedTemporaryFile(suffix='.png')
temp_file.seek(0)
temp_file.write(response)
temp_file.seek(0)
print(temp_file.name)
img_cover = open(temp_file.name, 'rb')
print(img_cover)
print(type(img_cover))
# crawling data 로 house info 채우기
data = {
'house_type': 'HO',
'name': listing['name'],
'description': '테스트용 집입니다.',
'room': listing['bedrooms'],
'bed': listing['beds'],
'bathroom': listing['bathrooms'],
'personnel': listing['person_capacity'],
'amenities': [],
'facilities': [1, 2, 3, 4, 5],
'minimum_check_in_duration': 1,
'maximum_check_in_duration': 3,
'maximum_check_in_range': 90,
'price_per_night': 100000,
'country': 'default',
'city': 'default',
'district': listing['localized_city'],
'dong': 'default',
'address1': 'default',
# 'address2': '희망빌라 2동 301호',
'latitude': listing['lat'],
'longitude': listing['lng'],
'disable_days': [
'2014-01-01',
'2014-02-01',
'2014-03-01',
'2014-04-01',
],
# 'img_cover': ContentFile(requests.get(listing['picture_urls'][0]).content),
'img_cover': img_cover,
# 'house_images': [
# ContentFile(requests.get(listing['picture_urls'][1]).content),
# ContentFile(requests.get(listing['picture_urls'][2]).content),
# ],
}
# host_user의 Token 값 header에 넣기
# 회원가입
# 로그인
user_data = {
'username': '******',
'password': '******'
}
# response = requests.post('/user/login/', user_data)
response = requests.post('http://localhost:8000/user/login/', user_data)
result = json.loads(response.text)
token = result['token']
headers = {
'Authorization': 'Token ' + token,
}
# response = requests.post('/house/', data, headers=headers)
response = requests.post('http://localhost:8000/house/', data, headers=headers)
print(response)
print(i+1)
img_cover.close()
temp_file.close()
def generate_export(export_type, xform, export_id=None, options=None,
retries=0):
"""
Create appropriate export object given the export type.
param: export_type
param: xform
params: export_id: ID of export object associated with the request
param: options: additional parameters required for the lookup.
binary_select_multiples: boolean flag
end: end offset
ext: export extension type
dataview_pk: dataview pk
group_delimiter: "/" or "."
query: filter_query for custom queries
remove_group_name: boolean flag
split_select_multiples: boolean flag
index_tag: ('[', ']') or ('_', '_')
"""
username = xform.user.username
id_string = xform.id_string
end = options.get("end")
extension = options.get("extension", export_type)
filter_query = options.get("query")
remove_group_name = options.get("remove_group_name", False)
start = options.get("start")
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
Export.GOOGLE_SHEETS_EXPORT: 'to_google_sheets',
}
if xform is None:
xform = XForm.objects.get(
user__username__iexact=username, id_string__iexact=id_string)
dataview = None
if options.get("dataview_pk"):
dataview = DataView.objects.get(pk=options.get("dataview_pk"))
records = dataview.query_data(dataview, all_data=True,
filter_query=filter_query)
total_records = dataview.query_data(dataview,
count=True)[0].get('count')
else:
records = query_data(xform, query=filter_query, start=start, end=end)
if filter_query:
total_records = query_data(xform, query=filter_query, start=start,
end=end, count=True)[0].get('count')
else:
total_records = xform.num_of_submissions
if isinstance(records, QuerySet):
records = records.iterator()
export_builder = ExportBuilder()
export_builder.TRUNCATE_GROUP_TITLE = True \
if export_type == Export.SAV_ZIP_EXPORT else remove_group_name
export_builder.GROUP_DELIMITER = options.get(
"group_delimiter", DEFAULT_GROUP_DELIMITER
)
export_builder.SPLIT_SELECT_MULTIPLES = options.get(
"split_select_multiples", True
)
export_builder.BINARY_SELECT_MULTIPLES = options.get(
"binary_select_multiples", False
)
export_builder.INCLUDE_LABELS = options.get('include_labels', False)
export_builder.INCLUDE_LABELS_ONLY = options.get(
'include_labels_only', False
)
export_builder.INCLUDE_HXL = options.get('include_hxl', False)
export_builder.INCLUDE_IMAGES \
= options.get("include_images", settings.EXPORT_WITH_IMAGE_DEFAULT)
export_builder.VALUE_SELECT_MULTIPLES = options.get(
'value_select_multiples', False)
export_builder.REPEAT_INDEX_TAGS = options.get(
"repeat_index_tags", DEFAULT_INDEX_TAGS
)
# 'win_excel_utf8' is only relevant for CSV exports
if 'win_excel_utf8' in options and export_type != Export.CSV_EXPORT:
del options['win_excel_utf8']
export_builder.set_survey(xform.survey, xform)
temp_file = NamedTemporaryFile(suffix=("." + extension))
columns_with_hxl = export_builder.INCLUDE_HXL and get_columns_with_hxl(
xform.survey_elements)
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
try:
func.__call__(
temp_file.name, records, username, id_string, filter_query,
start=start, end=end, dataview=dataview, xform=xform,
options=options, columns_with_hxl=columns_with_hxl,
total_records=total_records
)
except NoRecordsFoundError:
pass
except SPSSIOError as e:
export = get_or_create_export(export_id, xform, export_type, options)
export.error_message = str(e)
export.internal_status = Export.FAILED
export.save()
report_exception("SAV Export Failure", e, sys.exc_info())
return export
# generate filename
basename = "%s_%s" % (
id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f"))
if remove_group_name:
# add 'remove group name' flag to filename
basename = "{}-{}".format(basename, GROUPNAME_REMOVED_FLAG)
if dataview:
basename = "{}-{}".format(basename, DATAVIEW_EXPORT)
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(
username,
'exports',
id_string,
export_type,
filename)
# TODO: if s3 storage, make private - how will we protect local storage??
# seek to the beginning as required by storage classes
temp_file.seek(0)
export_filename = default_storage.save(file_path,
File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
export = get_or_create_export(export_id, xform, export_type, options)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# do not persist exports that have a filter
# Get URL of the exported sheet.
if export_type == Export.GOOGLE_SHEETS_EXPORT:
export.export_url = export_builder.url
# if we should create a new export is true, we should not save it
if start is None and end is None:
export.save()
return export
def generate_export(export_type,
extension,
username,
id_string,
export_id=None,
filter_query=None,
group_delimiter='/',
split_select_multiples=True,
binary_select_multiples=False,
sync_to_gsuit=False,
user=None):
"""
Create appropriate export object given the export type
"""
time.sleep(5)
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
Export.ANALYSER_EXPORT: 'to_analyser_export'
}
xform = XForm.objects.get(user__username__iexact=username,
id_string__exact=id_string)
# query mongo for the cursor
records = query_mongo(username, id_string, filter_query)
export_builder = ExportBuilder()
export_builder.GROUP_DELIMITER = group_delimiter
export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples
export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples
__version__ = "0"
try:
__version__ = filter_query['$and'][0]['__version__']
except Exception as e:
print(str(e))
if __version__:
survey = build_survey_from_history(xform, __version__)
if not survey:
export_builder.set_survey(xform.data_dictionary().survey)
else:
export_builder.set_survey(survey)
else:
export_builder.set_survey(xform.data_dictionary().survey)
prefix = slugify('{}_export__{}__{}'.format(export_type, username,
id_string))
temp_file = NamedTemporaryFile(prefix=prefix, suffix=("." + extension))
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
func.__call__(temp_file.name, records, username, id_string, filter_query)
# generate filename
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
if export_type == Export.ANALYSER_EXPORT:
# Analyser exports should be distinguished by more than just their file extension.
basename = '{}_ANALYSER_{}'.format(
id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(username, 'exports', id_string, export_type,
filename)
# TODO: if s3 storage, make private - how will we protect local storage??
storage = get_storage_class()()
# seek to the beginning as required by storage classes
print sync_to_gsuit, 'file_url--------->', temp_file, filter_query
try:
if sync_to_gsuit == True and '__version__' not in filter_query['$and'][
0]:
if not os.path.exists("media/forms/"):
os.makedirs("media/forms/")
temporarylocation = "media/forms/submissions_{}.xls".format(
id_string)
import shutil
shutil.copy(temp_file.name, temporarylocation)
fxf_form = FieldSightXF.objects.get(
pk=filter_query['$and'][0]['fs_project_uuid'])
upload_to_drive(temporarylocation,
str(fxf_form.id) + '_' + id_string, None,
fxf_form.project, user)
os.remove(temporarylocation)
except Exception as e:
print e.__dict__
# get or create export object
temp_file.seek(0)
export_filename = storage.save(file_path, File(temp_file, file_path))
dir_name, basename = os.path.split(export_filename)
temp_file.close()
if export_id:
export = Export.objects.get(id=export_id)
else:
fsxf = filter_query.values()[0]
# print("fsxf", fsxf)
export = Export(xform=xform, export_type=export_type, fsxf_id=fsxf)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# dont persist exports that have a filter
if filter_query is None:
export.save()
export.save()
return export
def test_file_truncates_namedtemporaryfile_name(self):
named_file = NamedTemporaryFile()
f = File(named_file)
self.assertEqual(f.name, os.path.basename(named_file.name))
"error":
"We failed to reach a server, reason: %s" %
e.reason
},
context_instance=RequestContext(request))
elif hasattr(e, "code"):
return render_to_response("error.html", {
"error":
"The remote server couldn't fulfill the request, HTTP error code %s"
% e.code
},
context_instance=RequestContext(
request))
# Store temp file.
url_temp = NamedTemporaryFile(delete=True)
url_temp.write(url.read())
url_temp.flush()
# Convert to File object.
url_file = File(url_temp).name
# Check content type.
mime = magic.Magic(mime=True)
content_type = mime.from_file(url_file)
if not check_allowed_content(content_type):
return render_to_response(
"error.html", {"error": "File type not supported"},
context_instance=RequestContext(request))
# Create analysis task.
def dataToImageFile(data):
image = NamedTemporaryFile(delete=False)
image.write(data)
image.flush()
return ImageFile(image)
def export_handler(request,
tag=None,
package_id=None,
branch=None,
version_guid=None,
block=None):
"""
The restful handler for exporting a course.
GET
html: return html page for import page
application/x-tgz: return tar.gz file containing exported course
json: not supported
Note that there are 2 ways to request the tar.gz file. The request header can specify
application/x-tgz via HTTP_ACCEPT, or a query parameter can be used (?_accept=application/x-tgz).
If the tar.gz file has been requested but the export operation fails, an HTML page will be returned
which describes the error.
"""
location = BlockUsageLocator(package_id=package_id,
branch=branch,
version_guid=version_guid,
block_id=block)
if not has_course_access(request.user, location):
raise PermissionDenied()
old_location = loc_mapper().translate_locator_to_location(location)
course_module = modulestore().get_item(old_location)
# an _accept URL parameter will be preferred over HTTP_ACCEPT in the header.
requested_format = request.REQUEST.get(
'_accept', request.META.get('HTTP_ACCEPT', 'text/html'))
export_url = location.url_reverse('export') + '?_accept=application/x-tgz'
if 'application/x-tgz' in requested_format:
name = old_location.name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
export_to_xml(modulestore('direct'), contentstore(), old_location,
root_dir, name, modulestore())
except SerializationError, e:
logging.exception(
'There was an error exporting course {0}. {1}'.format(
course_module.location, unicode(e)))
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_instance(
course_module.location.course_id, e.location)
parent_locs = modulestore().get_parent_locations(
failed_item.location, course_module.location.course_id)
if len(parent_locs) > 0:
parent = modulestore().get_item(parent_locs[0])
if parent.location.category == 'vertical':
unit = parent
except:
# if we have a nested exception, then we'll show the more generic error message
pass
unit_locator = loc_mapper().translate_location(
old_location.course_id, parent.location, False, True)
return render_to_response(
'export.html', {
'context_course':
course_module,
'in_err':
True,
'raw_err_msg':
str(e),
'failed_module':
failed_item,
'unit':
unit,
'edit_unit_url':
unit_locator.url_reverse("unit") if parent else "",
'course_home_url':
location.url_reverse("course"),
'export_url':
export_url
})
except Exception, e:
logging.exception(
'There was an error exporting course {0}. {1}'.format(
course_module.location, unicode(e)))
return render_to_response(
'export.html', {
'context_course': course_module,
'in_err': True,
'unit': None,
'raw_err_msg': str(e),
'course_home_url': location.url_reverse("course"),
'export_url': export_url
})
