def do_chart(request, slug): import re from django.http import HttpResponse from mxhelpers import get_rtt_dataset mx_obj = Mailserver.objects.get(slug=slug) graph_dataset = get_rtt_dataset(mx_obj, request.GET.get('range')) #rtt_labels = graph_dataset['rtt_labels'] data_temp = "t:%s|%s|%s" % (graph_dataset['sg_rtt_dataset'], graph_dataset['us_rtt_dataset'], graph_dataset['uk_rtt_dataset']) #au_rtt_list = re.sub("[\[\]]", '', str(graph_dataset['au_rtt_dataset'])) #au_rtt_dataset = graph_dataset['au_rtt_dataset'] #data_points = "t:%s" % re.sub("[\[\]\' ']", '', str(au_rtt_dataset)) data_points = re.sub("[\[\]\' ']", '', str(data_temp)) import urllib import urllib2 import time url = 'http://chart.apis.google.com/chart?' #data_points = 't:40,20,50,20,100|50,20,30,40,50' values = { 'cht': 'lc', 'chs': '700x200', 'chds': '0,%s' % graph_dataset['max_num'], 'chco': '96513c,008000,2865cc', 'chdlp': 'b', 'chxt': 'x,y', 'chxl': '0:|%s|1:|0|%s' % (graph_dataset['rtt_labels'], graph_dataset['rtt_y_labels']), 'chdl': 'Singapore|United Status|United Kingdom', 'chd': data_points } data = urllib.urlencode(values) req = urllib2.Request(url, data) #url_resp = urllib2.urlopen(req) #response = url_resp.read() from django.core.files import File from django.core.files.temp import NamedTemporaryFile img_temp = NamedTemporaryFile(delete=True) img_temp.write(urllib2.urlopen(req).read()) img_temp.flush() response = HttpResponse(File(img_temp), mimetype="image/png") return response
def create_eks_cluster(challenge): """ Called when Challenge is approved by the EvalAI admin calls the create_eks_nodegroup function Arguments: sender {type} -- model field called the post hook instance {<class 'django.db.models.query.QuerySet'>} -- instance of the model calling the post hook """ from .models import ChallengeEvaluationCluster for obj in serializers.deserialize("json", challenge): challenge_obj = obj.object cluster_name = "{0}-cluster".format(challenge_obj.title.replace(" ", "-")) if challenge_obj.approved_by_admin and challenge_obj.is_docker_based: client = get_boto3_client("eks", aws_keys) try: response = client.create_cluster( name=cluster_name, version="1.15", roleArn=settings.EKS_CLUSTER_ROLE_ARN, resourcesVpcConfig={ "subnetIds": [VPC_DICT["SUBNET_1"], VPC_DICT["SUBNET_2"]], "securityGroupIds": [VPC_DICT["SUBNET_SECURITY_GROUP"]], }, ) waiter = client.get_waiter("cluster_active") waiter.wait(name=cluster_name) # creating kubeconfig cluster = client.describe_cluster(name=cluster_name) cluster_cert = cluster["cluster"]["certificateAuthority"]["data"] cluster_ep = cluster["cluster"]["endpoint"] cluster_config = { "apiVersion": "v1", "kind": "Config", "clusters": [ { "cluster": { "server": str(cluster_ep), "certificate-authority-data": str(cluster_cert), }, "name": "kubernetes", } ], "contexts": [ { "context": {"cluster": "kubernetes", "user": "******"}, "name": "aws", } ], "current-context": "aws", "preferences": {}, "users": [ { "name": "aws", "user": { "exec": { "apiVersion": "client.authentication.k8s.io/v1alpha1", "command": "heptio-authenticator-aws", "args": ["token", "-i", cluster_name], } }, } ], } # Write in YAML. config_text = yaml.dump(cluster_config, default_flow_style=False) config_file = NamedTemporaryFile(delete=True) config_file.write(config_text.encode()) ChallengeEvaluationCluster.objects.create( challenge=challenge_obj, name=cluster_name, cluster_endpoint=cluster_ep, cluster_ssl=cluster_cert, ) # Creating nodegroup create_eks_nodegroup.delay(challenge, cluster_name) return response except ClientError as e: logger.exception(e) return
def process_locale_dir(self, locale_dir, files): """ Extract translatable literals from the specified files, creating or updating the POT file for a given locale directory. Use the xgettext GNU gettext utility. """ build_files = [] for translatable in files: if self.verbosity > 1: self.stdout.write('processing file %s in %s\n' % ( translatable.file, translatable.dirpath )) if self.domain not in ('djangojs', 'django'): continue build_file = self.build_file_class(self, self.domain, translatable) try: build_file.preprocess() except UnicodeDecodeError as e: self.stdout.write( 'UnicodeDecodeError: skipped file %s in %s (reason: %s)' % ( translatable.file, translatable.dirpath, e, ) ) continue build_files.append(build_file) if self.domain == 'djangojs': is_templatized = build_file.is_templatized args = [ 'xgettext', '-d', self.domain, '--language=%s' % ('C' if is_templatized else 'JavaScript',), '--keyword=gettext_noop', '--keyword=gettext_lazy', '--keyword=ngettext_lazy:1,2', '--keyword=pgettext:1c,2', '--keyword=npgettext:1c,2,3', '--output=-', ] elif self.domain == 'django': args = [ 'xgettext', '-d', self.domain, '--language=Python', '--keyword=gettext_noop', '--keyword=gettext_lazy', '--keyword=ngettext_lazy:1,2', '--keyword=ugettext_noop', '--keyword=ugettext_lazy', '--keyword=ungettext_lazy:1,2', '--keyword=pgettext:1c,2', '--keyword=npgettext:1c,2,3', '--keyword=pgettext_lazy:1c,2', '--keyword=npgettext_lazy:1c,2,3', '--output=-', ] else: return input_files = [bf.work_path for bf in build_files] with NamedTemporaryFile(mode='w+') as input_files_list: input_files_list.write('\n'.join(input_files)) input_files_list.flush() args.extend(['--files-from', input_files_list.name]) args.extend(self.xgettext_options) msgs, errors, status = popen_wrapper(args) if errors: if status != STATUS_OK: for build_file in build_files: build_file.cleanup() raise CommandError( 'errors happened while running xgettext on %s\n%s' % ('\n'.join(input_files), errors) ) elif self.verbosity > 0: # Print warnings self.stdout.write(errors) if msgs: if locale_dir is NO_LOCALE_DIR: file_path = os.path.normpath(build_files[0].path) raise CommandError( 'Unable to find a locale path to store translations for ' 'file %s' % file_path ) for build_file in build_files: msgs = build_file.postprocess_messages(msgs) potfile = os.path.join(locale_dir, '%s.pot' % self.domain) write_pot_file(potfile, msgs) for build_file in build_files: build_file.cleanup()
def handle(self, *args, **options): # Get default profile image image_url = '''http://cdn.scahw.com.au/cdn-1cfaa3a9e77a520/ imagevaultfiles/id_301664/cf_3/random-animals-16.jpg''' img_temp = NamedTemporaryFile(delete=True) img_temp.write(urlopen(image_url).read()) img_temp.flush() pic = File(img_temp) # Get default file to use for resume resume_url = '''http://writing.colostate.edu/guides/documents/resume/ functionalSample.pdf''' resume_temp = NamedTemporaryFile(delete=True) resume_temp.write(urlopen(resume_url).read()) resume_temp.flush() resume = File(resume_temp) # Demo hobbies to be used in following functions demo_hobbies = [ Hobby.objects.all().order_by('?').first(), Hobby.objects.all().order_by('?').first(), Hobby.objects.all().order_by('?').first(), Hobby.objects.all().order_by('?').first(), Hobby.objects.all().order_by('?').first(), ] # Create demo student user if not MyUser.objects.filter(email='*****@*****.**').exists(): student_demo = MyUser.objects.create_user( email='*****@*****.**', first_name='Student', last_name='Sample', password='******', gpa=3.45, degree_earned=MyUser.BACHELOR, year=MyUser.SENIOR, opp_sought=MyUser.FULL_TIME, ) student_demo.profile_pic.save("image_%s" % student_demo.username, pic), student_demo.resume.save("resume_%s" % student_demo.username, resume), student_demo.is_confirmed = True student_demo.account_type = MyUser.STUDENT student_demo.video = 'https://www.youtube.com/watch?v=_OBlgSz8sSM' student_demo.undergrad_uni = School.objects.get( name='Brown University') student_demo.undergrad_degree.add( Degree.objects.get(name='Business')) student_demo.save() for hobby in demo_hobbies: student_demo.hobbies.add(hobby) self.stdout.write( self.style.WARNING('Created demo student account.')) # Create demo employer user if not MyUser.objects.filter(email='*****@*****.**').exists(): employer_demo = MyUser.objects.create_user( email='*****@*****.**', first_name='Employer', last_name='Sample', password='******') employer_demo.is_confirmed = True employer_demo.account_type = MyUser.EMPLOYER employer_demo.save() self.stdout.write( self.style.WARNING('Created demo employer account.')) # Create users users = [['John', 'Doe'], ['Bill', 'Clarkson'], ['Jessica', 'Hall'], ['Franklin', 'Rose'], ['Bobby', 'Collins'], ['Fred', 'Flinstone'], ['Blake', 'Casper'], ['Marissa', 'Smiles'], ['Tyler', 'Simm'], ['Gina', 'Zentenial'], ['Michelle', 'Gregs'], ['Oscar', 'Behaves'], ['Heather', 'Hoolihan'], ['Scott', 'Dragonile'], ['Charlie', 'Bitfinger'], ['Pryia', 'Havalopolous'], ['Chris', 'Wildwood'], ['Jonathan', 'Newguinea'], ['Anne', 'Hathaway'], ['Brooke', 'Orchard']] if not MyUser.objects.filter(first_name=users[0][0]).exists(): for user in users: user = MyUser.objects.create_user( email='{}.{}@{}'.format( user[0], user[1], School.objects.all().order_by( '?').first().email).lower(), first_name=user[0], last_name=user[1], password='******', gpa=randint(int(2), int(100 * 4)) / 100.0, degree_earned=random.choice( [x[0] for x in MyUser.DEGREE_TYPES]), year=random.choice([x[0] for x in MyUser.YEAR_TYPES]), opp_sought=random.choice( [x[0] for x in MyUser.OPPORTUNITY_TYPES])) user.is_confirmed = True user.account_type = MyUser.STUDENT user.profile_pic.save("image_%s" % user.username, pic), user.resume.save("resume_%s" % user.username, resume), user.undergrad_uni = School.objects.all().order_by('?').first() user.undergrad_degree.add( Degree.objects.all().order_by('?').first()) user.save() for hobby in demo_hobbies: user.hobbies.add(hobby) self.stdout.write( self.style.WARNING('Created demo student accounts.')) # Create companies company_owners = [['Jeff', 'Bezos'], ['Pablo', 'Padre'], ['Jaime', 'Windell'], ['Naome', 'Watts']] if not MyUser.objects.filter(first_name=company_owners[0][0]).exists(): for user in company_owners: user = MyUser.objects.create_user( email='{}.{}@demo.com'.format(user[0], user[1]).lower(), first_name=user[0], last_name=user[1], password='******') user.is_confirmed = True user.account_type = MyUser.EMPLOYER user.save() self.stdout.write( self.style.WARNING('Created demo employer owner accounts.')) companies = [ ['Goldman Sachs', MyUser.objects.get(first_name='Jeff')], ['JPMorgan Chase', MyUser.objects.get(first_name='Pablo')], ['Morgan Stanley', MyUser.objects.get(first_name='Jaime')], ['American Express', MyUser.objects.get(first_name='Naome')], ['KPMG', MyUser.objects.get(first_name='Employer')], ] if not Company.objects.filter(name=companies[0][0]).exists(): for company in companies: Company.objects.create(user=company[1], name=company[0]) self.stdout.write( self.style.WARNING('Created demo employer accounts.')) # Create jobs description = '''Affert officiis consequat ut his. Vix cu ferri nostrum contentiones, in pri partem principes, pri id aliquid pericula. Nam atqui intellegat at. Et soleat perfecto mei, elitr abhorreant his an. Ex soleat habemus splendide mei, duo eu suas iisque, id eam justo argumentum. In sea cetero erroribus vituperatoribus. Dolorum senserit ad pri, no est nusquam definitiones. Has ipsum tincidunt ne. Eu mea aperiri euismod, vix in nominati inimicus. At simul adipiscing nec, dolore laboramus pro no, sea tale fierent ne. An corpora detracto corrumpit pri, epicurei intellegam quo in, dicit verterem id sit''' start = datetime.now() end = start + timedelta(days=21) contact_email = '*****@*****.**' jobs = [['Sales Associate', 'New York, NY'], ['IT Consultant', 'Hoboken, NJ'], ['Event Manager', 'Los Angeles, CA'], ['Senior Director', 'New York, NY'], ['EVP', 'Boston, MA'], ['Software Developer', 'Menlo Park, CA'], ['Marketing Associate', 'La Jolla, CA']] if not Job.objects.filter(title=jobs[0][0]).exists(): for job in jobs: company = Company.objects.order_by('?').first() Job.objects.create( user=company.user, company=company, title=job[0], contact_email=contact_email, description=description, location=job[1], list_date_start=start, list_date_end=end, ) self.stdout.write(self.style.WARNING('Created demo job listings.')) # Create applicants if not Applicant.objects.filter(user=MyUser.objects.get( first_name=users[0][0])).exists(): for applicant in users: _user = MyUser.objects.get(first_name=applicant[0]) Applicant.objects.create(user=_user, resume=settings.STATIC_URL + 'img/default-profile-pic.jpg', email=_user.email) self.stdout.write(self.style.WARNING('Created demo applicants.')) # Add applicants to jobs for applicant in Applicant.objects.all(): job = Job.objects.all().order_by('?').first() job.applicants.add(applicant) self.stdout.write( self.style.WARNING('Added applicants to demo job listings.')) self.stdout.write( self.style.SUCCESS('Successfully generated all sample data.'))
def test_international_homepage_serializer(rf, international_root_page, image): home_page = InternationalHomePageFactory( slug='international', parent= international_root_page, # This is tautological, but irrelevant here ) home_page.hero_subtitle = "THIS LEGACY FIELD SHOULD NOT BE USED" fake_video = make_test_video(duration=70, transcript='Test transcript note') with NamedTemporaryFile(delete=True) as img_tmp: fake_video.thumbnail.save('test.jpg', File(img_tmp)) fake_video.save() home_page.hero_video = fake_video home_page.hero_video.save() home_page.homepage_link_panels = [ ( 'link_panel', { 'title': 'panel one', 'supporting_text': 'panel one supporting text', 'link': { 'internal_link': home_page, # Circular link but doesn't matter in this test } }), ('link_panel', { 'title': 'panel two', 'supporting_text': 'panel two supporting text', 'link': { 'external_link': 'http://example.com/two/', } }), ] home_page.save() assert home_page.hero_subtitle is not None serializer = InternationalHomePageSerializer( instance=home_page, context={'request': rf.get('/')}) assert serializer.data['hero_title'] == home_page.hero_title homepage_link_panels = serializer.data['homepage_link_panels'] assert homepage_link_panels[0]['type'] == 'link_panel' assert homepage_link_panels[0]['value'] == { 'title': 'panel one', 'supporting_text': 'panel one supporting text', 'link': home_page.url, } assert homepage_link_panels[1]['type'] == 'link_panel' assert homepage_link_panels[1]['value'] == { 'title': 'panel two', 'supporting_text': 'panel two supporting text', 'link': 'http://example.com/two/', } assert serializer.data['hero_video']['title'] == 'Test file' assert serializer.data['hero_video'][ 'transcript'] == 'Test transcript note' assert serializer.data['hero_video'][ 'thumbnail'] == fake_video.thumbnail.url assert serializer.data['hero_video']['sources'][0]['src'] == fake_video.url # confirm the legacy fields are not exposed: for example_field_name in [ 'hero_subtitle', 'hero_cta_text', 'hero_cta_link', 'hero_image', 'brexit_banner_text', 'invest_title', 'invest_content', 'invest_image', # there are more, but these are useful smoke tests 'case_study_image', 'case_study_title', 'case_study_text', 'case_study_cta_text', 'case_study_cta_link', ]: assert example_field_name not in serializer.data
def generate_export(export_type, extension, username, id_string, export_id=None, filter_query=None, group_delimiter='/', split_select_multiples=True, binary_select_multiples=False): """ Create appropriate export object given the export type """ export_type_func_map = { Export.XLS_EXPORT: 'to_xls_export', Export.CSV_EXPORT: 'to_flat_csv_export', Export.CSV_ZIP_EXPORT: 'to_zipped_csv', Export.SAV_ZIP_EXPORT: 'to_zipped_sav', Export.ANALYSER_EXPORT: 'to_analyser_export' } xform = XForm.objects.get(user__username__iexact=username, id_string__exact=id_string) # query mongo for the cursor records = query_mongo(username, id_string, filter_query) export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = group_delimiter export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples export_builder.set_survey(xform.data_dictionary().survey) prefix = slugify('{}_export__{}__{}'.format(export_type, username, id_string)) temp_file = NamedTemporaryFile(prefix=prefix, suffix=("." + extension)) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) func.__call__(temp_file.name, records, username, id_string, filter_query) # generate filename basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) if export_type == Export.ANALYSER_EXPORT: # Analyser exports should be distinguished by more than just their file extension. basename = '{}_ANALYSER_{}'.format( id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + extension # check filename is unique while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) file_path = os.path.join(username, 'exports', id_string, export_type, filename) # TODO: if s3 storage, make private - how will we protect local storage?? storage = get_storage_class()() # seek to the beginning as required by storage classes temp_file.seek(0) export_filename = storage.save(file_path, File(temp_file, file_path)) temp_file.close() dir_name, basename = os.path.split(export_filename) # get or create export object if export_id: export = Export.objects.get(id=export_id) else: export = Export(xform=xform, export_type=export_type) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL # dont persist exports that have a filter if filter_query is None: export.save() return export
def import_entries(self, feed_entries): """Import entries""" for feed_entry in feed_entries: self.write_out('> %s... ' % feed_entry.title) if feed_entry.get('published_parsed'): creation_date = datetime(*feed_entry.published_parsed[:6]) if settings.USE_TZ: creation_date = timezone.make_aware( creation_date, timezone.utc) else: creation_date = timezone.now() slug = slugify(feed_entry.title)[:255] if Entry.objects.filter(creation_date__year=creation_date.year, creation_date__month=creation_date.month, creation_date__day=creation_date.day, slug=slug): self.write_out( self.style.NOTICE('SKIPPED (already imported)\n')) continue categories = self.import_categories(feed_entry) entry_dict = { 'title': feed_entry.title[:255], 'content': feed_entry.description, 'excerpt': feed_entry.get('summary'), 'status': PUBLISHED, 'creation_date': creation_date, 'start_publication': creation_date, 'last_update': timezone.now(), 'slug': slug } if not entry_dict['excerpt'] and self.auto_excerpt: entry_dict['excerpt'] = Truncator( strip_tags(feed_entry.description)).words(50) if self.tags: entry_dict['tags'] = self.import_tags(categories) entry = Entry(**entry_dict) entry.save() entry.categories.add(*categories) entry.sites.add(self.SITE) if self.image_enclosure: for enclosure in feed_entry.enclosures: if ('image' in enclosure.get('type') and enclosure.get('href')): img_tmp = NamedTemporaryFile(delete=True) img_tmp.write(urlopen(enclosure['href']).read()) img_tmp.flush() entry.image.save(os.path.basename(enclosure['href']), File(img_tmp)) break if self.default_author: entry.authors.add(self.default_author) elif feed_entry.get('author_detail'): try: author = Author.objects.create_user( slugify(feed_entry.author_detail.get('name')), feed_entry.author_detail.get('email', '')) except IntegrityError: author = Author.objects.get( username=slugify(feed_entry.author_detail.get('name'))) entry.authors.add(author) self.write_out(self.style.ITEM('OK\n'))
def run_diarios(self): http = urllib3.PoolManager() def reset_id_model(model): query = """SELECT setval(pg_get_serial_sequence('"%(app_model_name)s"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "%(app_model_name)s"; """ % { 'app_model_name': _get_registration_key(model) } with connection.cursor() as cursor: cursor.execute(query) # get all the rows as a list rows = cursor.fetchall() print(rows) tipo = Tipolei.objects.get(id=5) docs = Documento.objects.filter(assuntos__tipo=tipo, publicado=True) docs = docs.order_by('data_lei') tipo_de_diario = TipoDeDiario.objects.get(pk=1) # DiarioOficial.objects.all().delete() for doc in docs: try: diario = DiarioOficial.objects.get(id=doc.id) except: diario = DiarioOficial() diario.id = doc.id diario.tipo = tipo_de_diario diario.descricao = doc.epigrafe diario.edicao = doc.numero diario.data = doc.data_lei diario.save() normas = NormaJuridica.objects.filter( veiculo_publicacao=diario.edicao) if normas.exists(): diario.normas.add(*list(normas)) if diario.arquivo: print('PULANDO', diario.id, DiarioOficial._meta.object_name) continue sleep(1) print('Fazendo', diario.id, DiarioOficial._meta.object_name) url = ('http://168.228.184.68:8580/portal/' 'downloadFile.pdf?sv=2&id=%s') % diario.id request = None try: request = http.request('GET', url, timeout=10.0, retries=False) if not request: continue if request.status == 404: print(diario.pk, "não possui arquivo...") continue if not request.data or len(request.data) == 0: continue temp = NamedTemporaryFile(delete=True) temp.write(request.data) temp.flush() try: name_file = '%s_diario_oficial_%s.pdf' % (diario.id, diario.edicao) diario.arquivo.save(name_file, File(temp), save=True) except Exception as e: print(e) except: print(diario.pk, "erro...") reset_id_model(DiarioOficial) print(doc.id, doc.numero)
def __buscar_datos(self,soup,url): fotos = soup.find_all('img', attrs={'class': 'foto'}) for foto in fotos: numinv = foto['title'].split()[-1] if numinv not in registros: registros.append(numinv) codigo = self.__obtener_codigo(numinv) titulo = "(?)" dimensiones = (0, 0) fechaingreso = None datacion = "" descripcion = "" observaciones = "" soporte = "" iconografias = [] tecnicas = [] all_row = soup.find_all('th', attrs={'class': 'tabla1TituloMB'}) for row in all_row: if 'Título' in row.text.encode('utf-8').strip(): if numinv in row.parent.td['headers'][0]: titulo = row.parent.td.text.encode('utf-8').strip() if 'Dimensiones' in row.text.encode('utf-8').strip(): if numinv in row.parent.td['headers'][0]: aux = row.parent.td.text.encode('utf-8').strip() dimensiones = self.__obtener_dimensiones(aux) if 'Datación' in row.text.encode('utf-8').strip(): if numinv in row.parent.td['headers'][0]: datacion = row.parent.td.text.encode('utf-8').strip() datacion = re.sub('[(]{1}.*[)]{1}','',datacion).strip() if 'Descripción' in row.text.encode('utf-8').strip(): if numinv in row.parent.td['headers'][0]: descripcion = row.parent.td.text.encode('utf-8').strip() if 'Observaciones' in row.text.encode('utf-8').strip(): if numinv in row.parent.td['headers'][0]: observaciones = row.parent.td.text.encode('utf-8').strip() if 'Materia/Soporte' in row.text.encode('utf-8').strip(): if numinv in row.parent.td['headers'][0]: soporte = row.parent.td.text.encode('utf-8').strip() soporte = re.sub('[\[]{1}.*[\]]{1}','',soporte).strip() if 'Iconografia' in row.text.encode('utf-8').strip(): if numinv in row.parent.td['headers'][0]: iconografia = row.parent.td.text.encode('utf-8').strip() iconografias = iconografia.replace(' ',';').split(';') if 'Técnica' in row.text.encode('utf-8').strip(): if numinv in row.parent.td['headers'][0]: tecnica = row.parent.td.text.encode('utf-8').strip() tecnicas = tecnica.replace(' ',';').split(';') img_url = "http://ceres.mcu.es/pages/Viewer?accion=42&Museo=&AMuseo=MCA&Ninv="+numinv if debug: print '\n\033[1m\033[33m' + codigo[0] + codigo[1] + "\033[0m" print 'Título: ' + fix_string(titulo) print 'Altura: ' + str(dimensiones[0]) + ' cm' print 'Anchura: ' + str(dimensiones[1]) + ' cm' print 'Datación: ' + fix_string(datacion) print 'Descripción: ' + fix_string(descripcion[:75]) + "..." print 'Observaciones: ' + fix_string(observaciones[:75]) + "..." print 'Soporte: ' + fix_string(soporte) if iconografias: print 'Iconografías: ' for iconografia in iconografias: print fix_string('\t'+iconografia[:30].strip()) if tecnicas: print 'Técnicas: ' for tecnica in tecnicas: print fix_string('\t'+re.sub('[\[]{1}.*[\]]{1}','',tecnica)[:30].strip()) print 'Imagen: ' + img_url else: try: print "\033[33m[ ]\033[0m " + codigo[0] + codigo[1] + "\r", sys.stdout.flush() s = Soporte.objects.filter(nombre=fix_string(soporte)) if len(s): objSoporte = s[0] else: if len(soporte) and len(soporte) <= 30: objSoporte = Soporte(nombre=fix_string(soporte)) objSoporte.save() else: objSoporte = None setIconografias = [] for iconografia in iconografias: if len(iconografia) and len(iconografia) <= 30: i = Iconografia.objects.filter(nombre=fix_string(iconografia.strip())) if len(i): objIconografia = i[0] else: objIconografia = Iconografia(nombre=fix_string(iconografia.strip())) objIconografia.save() setIconografias.append(objIconografia) setTecnicas = [] for tecnica in tecnicas: tecnica = re.sub('[\[]{1}.*[\]]{1}','',tecnica) if len(tecnica) and len(tecnica) <= 30: t = Tecnica.objects.filter(nombre=fix_string(tecnica.strip())) if len(t): objTecnica = t[0] else: objTecnica = Tecnica(nombre=fix_string(tecnica.strip())) objTecnica.save() setTecnicas.append(objTecnica) arq = Arqueologia.objects.filter(numinv=codigo[1]) if len(arq): arq[0].delete() ba = Bellasartes(numinv=codigo[1]) ba.save_base() ba.titulo = fix_string(titulo).split(";")[0][:100] ba.codigo = fix_string(codigo[0]) ba.altura = dimensiones[0] ba.ancho = dimensiones[1] ba.datacion = fix_string(datacion).replace('=','-') ba.descripcion = fix_string(descripcion) ba.observaciones = fix_string(observaciones) ba.fechaingreso = fechaingreso ba.iconografia.set(setIconografias) ba.tecnica.set(setTecnicas) try: ba.soporte = objSoporte except: pass try: img_temp = NamedTemporaryFile(delete=True) img_temp.write(urllib2.urlopen(img_url).read()) img_temp.flush() ba.anverso.save(codigo[0]+'.jpg', File(img_temp)) except: pass ba.save() print "\033[32m[+]\033[0m" except: print "\033[31m[x]\033[0m"
def test_to_xls_export_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) xls_file = NamedTemporaryFile(suffix='.xls') filename = xls_file.name export_builder.to_xls_export(filename, self.data) xls_file.seek(0) wb = load_workbook(filename) # check that we have childrens_survey, children, children_cartoons # and children_cartoons_characters sheets expected_sheet_names = ['childrens_survey', 'children', 'children_cartoons', 'children_cartoons_characters'] self.assertEqual(wb.get_sheet_names(), expected_sheet_names) # check header columns main_sheet = wb.get_sheet_by_name('childrens_survey') expected_column_headers = [ u'name', u'age', u'geo/geolocation', u'geo/_geolocation_latitude', u'geo/_geolocation_longitude', u'geo/_geolocation_altitude', u'geo/_geolocation_precision', u'tel/tel.office', u'tel/tel.mobile', u'_id', u'meta/instanceID', u'_uuid', u'_submission_time', u'_index', u'_parent_index', u'_parent_table_name', u'_tags', '_notes'] column_headers = [c[0].value for c in main_sheet.columns] self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) childrens_sheet = wb.get_sheet_by_name('children') expected_column_headers = [ u'children/name', u'children/age', u'children/fav_colors', u'children/fav_colors/red', u'children/fav_colors/blue', u'children/fav_colors/pink', u'children/ice.creams', u'children/ice.creams/vanilla', u'children/ice.creams/strawberry', u'children/ice.creams/chocolate', u'_id', u'_uuid', u'_submission_time', u'_index', u'_parent_index', u'_parent_table_name', u'_tags', '_notes'] column_headers = [c[0].value for c in childrens_sheet.columns] self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) cartoons_sheet = wb.get_sheet_by_name('children_cartoons') expected_column_headers = [ u'children/cartoons/name', u'children/cartoons/why', u'_id', u'_uuid', u'_submission_time', u'_index', u'_parent_index', u'_parent_table_name', u'_tags', '_notes'] column_headers = [c[0].value for c in cartoons_sheet.columns] self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) characters_sheet = wb.get_sheet_by_name('children_cartoons_characters') expected_column_headers = [ u'children/cartoons/characters/name', u'children/cartoons/characters/good_or_evil', u'_id', u'_uuid', u'_submission_time', u'_index', u'_parent_index', u'_parent_table_name', u'_tags', '_notes'] column_headers = [c[0].value for c in characters_sheet.columns] self.assertEqual(sorted(column_headers), sorted(expected_column_headers)) xls_file.close()
def post(self, request, *args, **kwargs): solution = get_object_or_404(Solution, pk=kwargs.pop('pk')) solver = Solver() status = solver.status(solution.submission_id) if status == Solver.SUCCESS: info = solver.info(solution.submission_id) solution.objects_in_field = ', '.join(info['objects_in_field']) solution.ra = "%.3f" % info['calibration']['ra'] solution.dec = "%.3f" % info['calibration']['dec'] solution.orientation = "%.3f" % info['calibration']['orientation'] solution.radius = "%.3f" % info['calibration']['radius'] # Get the images 'w' and adjust pixscale if solution.content_object: w = solution.content_object.w pixscale = info['calibration']['pixscale'] if w and pixscale: hd_w = settings.THUMBNAIL_ALIASES['']['hd']['size'][0] if hd_w > w: hd_w = w ratio = hd_w / float(w) corrected_scale = float(pixscale) * ratio solution.pixscale = "%.3f" % corrected_scale else: solution.pixscale = None try: target = solution.content_type.get_object_for_this_type( pk=solution.object_id) except solution.content_type.model_class().DoesNotExist: # Target image was deleted meanwhile context = {'status': Solver.FAILED} return HttpResponse(simplejson.dumps(context), content_type='application/json') # Annotate image annotations_obj = solver.annotations(solution.submission_id) solution.annotations = simplejson.dumps(annotations_obj) annotator = Annotator(solution) annotated_image = annotator.annotate() filename, ext = os.path.splitext(target.image_file.name) annotated_filename = "%s-%d%s" % (filename, int(time.time()), ext) if annotated_image: solution.image_file.save(annotated_filename, annotated_image) # Get sky plot image url = solver.sky_plot_zoom1_image_url(solution.submission_id) if url: img = NamedTemporaryFile(delete=True) img.write(urllib2.urlopen(url).read()) img.flush() img.seek(0) f = File(img) try: solution.skyplot_zoom1.save(target.image_file.name, f) except IntegrityError: pass solution.status = status solution.save() context = {'status': solution.status} return HttpResponse(simplejson.dumps(context), content_type='application/json')
def test_zipped_csv_export_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) temp_zip_file = NamedTemporaryFile(suffix='.zip') export_builder.to_zipped_csv(temp_zip_file.name, self.data) temp_zip_file.seek(0) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(temp_zip_file.name, "r") zip_file.extractall(temp_dir) zip_file.close() temp_zip_file.close() # generate data to compare with index = 1 indices = {} survey_name = survey.name outputs = [] for d in self.data: outputs.append( dict_to_joined_export(d, index, indices, survey_name)) index += 1 # check that each file exists self.assertTrue( os.path.exists( os.path.join(temp_dir, "{0}.csv".format(survey.name)))) with open( os.path.join( temp_dir, "{0}.csv".format(survey.name))) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file with open(_logger_fixture_path( 'csvs', 'childrens_survey.csv')) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) self.assertTrue( os.path.exists( os.path.join(temp_dir, "children.csv"))) with open(os.path.join(temp_dir, "children.csv")) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file with open(_logger_fixture_path( 'csvs', 'children.csv')) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) self.assertTrue( os.path.exists( os.path.join(temp_dir, "children_cartoons.csv"))) with open(os.path.join(temp_dir, "children_cartoons.csv")) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file with open(_logger_fixture_path( 'csvs', 'children_cartoons.csv')) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) self.assertTrue( os.path.exists( os.path.join(temp_dir, "children_cartoons_characters.csv"))) with open(os.path.join( temp_dir, "children_cartoons_characters.csv")) as csv_file: reader = csv.reader(csv_file) rows = [r for r in reader] # open comparison file with open(_logger_fixture_path( 'csvs', 'children_cartoons_characters.csv')) as fixture_csv: fixture_reader = csv.reader(fixture_csv) expected_rows = [r for r in fixture_reader] self.assertEqual(rows, expected_rows) shutil.rmtree(temp_dir)
def test_partial_iati_import(self): """ Test an IATI import that ignores certain elements Ignore the data if the akvo:import attribute set to falsey values, i.e. "false", "no", "f" or "0". Elements that support this are: budget, contact-info, humanitarian-scope, legacy-data, location, participating-org, planned-disbursement, policy-marker, recipient-country, recipient-region, related-activity, result, sector and transaction """ #import a project iati_v2_import = IatiImport.objects.create(label="Test IATI v2 import", user=self.user) iati_v2_xml_file = NamedTemporaryFile(delete=True) iati_v2_xml_file.write(IATI_V2_STRING) iati_v2_xml_file.flush() iati_v2_import_job = IatiImportJob.objects.create( iati_import=iati_v2_import, iati_xml_file=File(iati_v2_xml_file)) iati_v2_import_job.run() project_v2 = Project.objects.get( iati_activity_id="NL-KVK-0987654321-v2") self.assertEqual(project_v2.contacts.count(), 1) contact_info = project_v2.contacts.all()[0] self.assertEqual(contact_info.organisation, "Agency A") self.assertEqual(contact_info.department, "Department B") self.assertEqual(project_v2.locations.count(), 2) location_1 = project_v2.locations.get(reference="AF-KAN") location_2 = project_v2.locations.get(reference="KH-PNH") self.assertEqual(location_1.location_code, "1453782") self.assertEqual(location_2.location_code, "1821306") # do a new import to the same project, with contact and location elements having # akvo:import="false" partial_import = IatiImport.objects.create( label="Test partial IATI import", user=self.user) partial_import_xml_file = NamedTemporaryFile(delete=True) partial_import_xml_file.write(IATI_PARTIAL_IMPORT) partial_import_xml_file.flush() partial_import_job = IatiImportJob.objects.create( iati_import=partial_import, iati_xml_file=File(partial_import_xml_file)) partial_import_job.run() project_partial_import = Project.objects.get( iati_activity_id="NL-KVK-0987654321-v2") self.assertIsInstance(project_partial_import, Project) # Assert that no data has changed, even if the XML has self.assertEqual(project_v2.results.count(), 1) result_1 = project_v2.results.get(title="Result title") self.assertEqual(result_1.indicators.count(), 1) self.assertEqual( result_1.indicators.all()[0].periods.all()[0].actual_value, u'11') self.assertEqual(project_v2.contacts.count(), 1) contact_info = project_v2.contacts.all()[0] self.assertEqual(contact_info.organisation, "Agency A") self.assertEqual(contact_info.department, "Department B") self.assertEqual(project_v2.locations.count(), 2) location_1 = project_v2.locations.get(reference="AF-KAN") location_2 = project_v2.locations.get(reference="KH-PNH") self.assertEqual(location_1.location_code, "1453782") self.assertEqual(location_2.location_code, "1821306") self.assertEqual(project_v2.humanitarian_scopes.count(), 2) humanitarian_scope_1 = project_v2.humanitarian_scopes.get( vocabulary="1-2") humanitarian_scope_2 = project_v2.humanitarian_scopes.get( vocabulary="99") self.assertEqual(humanitarian_scope_1.code, "2015-000050") self.assertEqual(humanitarian_scope_2.vocabulary_uri, "http://example.com/vocab.html") # three participating orgs, and one reporting org self.assertEqual(project_v2.partners.count(), 4) self.assertEqual(project_v2.planned_disbursements.count(), 2) self.assertEqual(project_v2.policy_markers.count(), 3) self.assertEqual(project_v2.recipient_countries.count(), 2) self.assertEqual(project_v2.recipient_regions.count(), 3) related_project_1 = project_v2.related_projects.get( related_iati_id="AA-AAA-123456789-6789") self.assertIsInstance(related_project_1, RelatedProject) self.assertEqual(project_v2.sectors.count(), 3) self.assertEqual(project_v2.transactions.count(), 1) self.assertEqual(project_v2.budget_items.count(), 1)
def handle(self, **options): if not settings.MEDIA_ROOT: raise ImproperlyConfigured( 'Please set MEDIA_ROOT in your settings file') remote_url = options['remote_url'] try: val = URLValidator() val(remote_url) except ValidationError: raise CommandError('Please enter a valid URL') exercise_api = "{0}/api/v2/exercise/?limit=999" image_api = "{0}/api/v2/exerciseimage/?exercise={1}" thumbnail_api = "{0}/api/v2/exerciseimage/{1}/thumbnails/" headers = { 'User-agent': default_user_agent('wger/{} + requests'.format(get_version())) } # Get all exercises result = requests.get(exercise_api.format(remote_url), headers=headers).json() for exercise_json in result['results']: exercise_name = exercise_json['name'].encode('utf-8') exercise_uuid = exercise_json['uuid'] exercise_id = exercise_json['id'] self.stdout.write('') self.stdout.write( u"*** Processing {0} (ID: {1}, UUID: {2})".format( exercise_name, exercise_id, exercise_uuid)) try: exercise = Exercise.objects.get(uuid=exercise_uuid) except Exercise.DoesNotExist: self.stdout.write( ' Remote exercise not found in local DB, skipping...') continue # Get all images images = requests.get(image_api.format(remote_url, exercise_id), headers=headers).json() if images['count']: for image_json in images['results']: image_id = image_json['id'] result = requests.get(thumbnail_api.format( remote_url, image_id), headers=headers).json() image_name = os.path.basename(result['original']) self.stdout.write(' Fetching image {0} - {1}'.format( image_id, image_name)) try: image = ExerciseImage.objects.get(pk=image_id) self.stdout.write( ' --> Image already present locally, skipping...' ) continue except ExerciseImage.DoesNotExist: self.stdout.write( ' --> Image not found in local DB, creating now...' ) image = ExerciseImage() image.pk = image_id # Save the downloaded image, see link for details # http://stackoverflow.com/questions/1308386/programmatically-saving-image-to- retrieved_image = requests.get(result['original'], headers=headers) img_temp = NamedTemporaryFile(delete=True) img_temp.write(retrieved_image.content) img_temp.flush() image.exercise = exercise image.is_main = image_json['is_main'] image.status = image_json['status'] image.image.save( os.path.basename(image_name), File(img_temp), ) image.save() else: self.stdout.write( ' No images for this exercise, nothing to do')
def to_zipped_csv(self, path, data, *args): def write_row(row, csv_writer, fields): csv_writer.writerow( [encode_if_str(row, field) for field in fields]) csv_defs = {} for section in self.sections: csv_file = NamedTemporaryFile(suffix=".csv") csv_writer = csv.writer(csv_file) csv_defs[section['name']] = { 'csv_file': csv_file, 'csv_writer': csv_writer } # write headers for section in self.sections: fields = [element['title'] for element in section['elements']]\ + self.EXTRA_FIELDS csv_defs[section['name']]['csv_writer'].writerow( [f.encode('utf-8') for f in fields]) index = 1 indices = {} survey_name = self.survey.name for d in data: # decode mongo section names joined_export = dict_to_joined_export(d, index, indices, survey_name) output = ExportBuilder.decode_mongo_encoded_section_names( joined_export) # attach meta fields (index, parent_index, parent_table) # output has keys for every section if survey_name not in output: output[survey_name] = {} output[survey_name][INDEX] = index output[survey_name][PARENT_INDEX] = -1 for section in self.sections: # get data for this section and write to csv section_name = section['name'] csv_def = csv_defs[section_name] fields = [element['xpath'] for element in section['elements'] ] + self.EXTRA_FIELDS csv_writer = csv_def['csv_writer'] # section name might not exist within the output, e.g. data was # not provided for said repeat - write test to check this row = output.get(section_name, None) if type(row) == dict: write_row(self.pre_process_row(row, section), csv_writer, fields) elif type(row) == list: for child_row in row: write_row(self.pre_process_row(child_row, section), csv_writer, fields) index += 1 # write zipfile with ZipFile(path, 'w') as zip_file: for section_name, csv_def in csv_defs.iteritems(): csv_file = csv_def['csv_file'] csv_file.seek(0) zip_file.write(csv_file.name, "_".join(section_name.split("/")) + ".csv") # close files when we are done for section_name, csv_def in csv_defs.iteritems(): csv_def['csv_file'].close()
def parse(headers, url) -> (Product, ProductPage): resp = requests.get(url, headers=headers) content = str(resp.content) parser = DetailParser(content) detail = parser.parse() product_title = detail.get('title') product_reviews_number = detail.get('reviews') price_elements = parser.selector.css('#priceblock_saleprice') or parser.selector.css('.offer-price') product_price = None if len(price_elements) > 0: product_price = price_elements[0].root.text product = Product( product_title=product_title.strip().replace('\n', '').replace('\\n', '').replace(' ', ''), url=url, price=float(product_price.replace('$', '')), rating_amount=float(product_reviews_number or 0) ) def get_number(str_with_num) -> List[float]: return [item[1] if item[1] else item[0] for item in re.findall('(\d+\.\d+)|(\d+)', value)] def get_first_float(numbers: List[float]) -> float: return float(str(numbers[0])) if len(numbers) else None for key, value in detail['details'].items(): if 'Shipping Weight' in key: numbers = get_number(value) product.shipping_weight = get_first_float(numbers) elif 'Weight' in key: numbers = get_number(value) product.weight = get_first_float(numbers) elif 'Dimensions' in key: numbers = get_number(value) product.dim_x, product.dim_y, product.dim_z = numbers elif 'model number' in key: product.model_number = value.replace(' ', '').replace('\n', '').replace('\\n', '') elif 'ASIN' in key: product.asin = value.replace(' ', '').replace('\n', '').replace('\\n', '') elif 'Best Sellers Rank' in key: numbers = get_number(value) product.bsr = get_first_float(numbers) content = ProductPage(page_html=content) images = [] try: image_responses = parser.get_image_responses() for url, response in image_responses: hash_sum = md5() image_temp_file = NamedTemporaryFile(delete=True) for chunk in response.iter_content(1024): if not chunk: break hash_sum.update(chunk) image_temp_file.write(chunk) dig = hash_sum.hexdigest() image_temp_file.flush() filename = f'{product.asin}_{dig}' temp_file = File(image_temp_file, name=filename) image = ProductImage( name=filename, hash=dig, product=product, ) image.file.save(filename, temp_file) images.append(image) except: pass return product, content, images
def to_zipped_sav(self, path, data, *args): def write_row(row, csv_writer, fields): sav_writer.writerow( [encode_if_str(row, field, True) for field in fields]) sav_defs = {} # write headers for section in self.sections: fields = [element['title'] for element in section['elements']]\ + self.EXTRA_FIELDS c = 0 var_labels = {} var_names = [] tmp_k = {} for field in fields: c += 1 var_name = 'var%d' % c var_labels[var_name] = field var_names.append(var_name) tmp_k[field] = var_name var_types = dict( [(tmp_k[element['title']], 0 if element['type'] in ['decimal', 'int'] else 255) for element in section['elements']] + [(tmp_k[item], 0 if item in ['_id', '_index', '_parent_index'] else 255) for item in self.EXTRA_FIELDS]) sav_file = NamedTemporaryFile(suffix=".sav") sav_writer = SavWriter(sav_file.name, varNames=var_names, varTypes=var_types, varLabels=var_labels, ioUtf8=True) sav_defs[section['name']] = { 'sav_file': sav_file, 'sav_writer': sav_writer } index = 1 indices = {} survey_name = self.survey.name for d in data: # decode mongo section names joined_export = dict_to_joined_export(d, index, indices, survey_name) output = ExportBuilder.decode_mongo_encoded_section_names( joined_export) # attach meta fields (index, parent_index, parent_table) # output has keys for every section if survey_name not in output: output[survey_name] = {} output[survey_name][INDEX] = index output[survey_name][PARENT_INDEX] = -1 for section in self.sections: # get data for this section and write to csv section_name = section['name'] sav_def = sav_defs[section_name] fields = [element['xpath'] for element in section['elements'] ] + self.EXTRA_FIELDS sav_writer = sav_def['sav_writer'] row = output.get(section_name, None) if type(row) == dict: write_row(self.pre_process_row(row, section), sav_writer, fields) elif type(row) == list: for child_row in row: write_row(self.pre_process_row(child_row, section), sav_writer, fields) index += 1 for section_name, sav_def in sav_defs.iteritems(): sav_def['sav_writer'].closeSavFile(sav_def['sav_writer'].fh, mode='wb') # write zipfile with ZipFile(path, 'w') as zip_file: for section_name, sav_def in sav_defs.iteritems(): sav_file = sav_def['sav_file'] sav_file.seek(0) zip_file.write(sav_file.name, "_".join(section_name.split("/")) + ".sav") # close files when we are done for section_name, sav_def in sav_defs.iteritems(): sav_def['sav_file'].close()
def classify_api(request): data = {"success": False} clean_directory() if request.method == "POST": model = request.POST.get("model", None) if model == 'imagenet': tmp_f = NamedTemporaryFile() tmp_adver = NamedTemporaryFile() tmp_adver_gif = NamedTemporaryFile() if request.FILES.get("image", None) is not None: image_request = request.FILES["image"] image_bytes = image_request.read() image.save(tmp_f, image.format) elif request.POST.get("image64", None) is not None: base64_data = request.POST.get("image64", None).split(',', 1)[1] plain_data = base64.b64decode(base64_data) image = Image.open(io.BytesIO(plain_data)) image.save( os.path.join(current_dir, 'imagenet/dataset/images/testtest.png')) tmp_f.write(plain_data) tmp_f.close() # Backend session for attack print('Building Backend Session.') K.set_learning_phase(0) sess = tf.Session() backend.set_session(sess) # Image preprocess print('Modifying image') x = np.expand_dims(preprocess(image.resize((299, 299))), axis=0) img_shape = [1, 299, 299, 3] x_input = tf.placeholder(tf.float32, shape=img_shape) # Define model d = discriminator() # Prediction of original image print('prediction of original image') classify_result = get_predictions(d, x, 10) # Select attack algorithm and iteration attack_algorithm = request.POST.get("attack", None) n = int(request.POST.get("iterate", None)) # Start attack result, attack_speed = attack(attack_algorithm, n, d, x_input, x, sess) print("attack speed: %s seconds" % (round(attack_speed, 5))) print('original image:', classify_result[0][1]) print('adversarial example is classified by', result[0][1]) # Print image to web site with open( os.path.join(current_dir, 'imagenet/output/testtest.png'), 'rb') as img_file: img_str = base64.b64encode(img_file.read()) tmp_adver.write(base64.b64decode(img_str)) tmp_adver.close() with open( os.path.join(current_dir, 'imagenet/output/testtest.gif'), 'rb') as gif_file: gif_str = base64.b64encode(gif_file.read()) tmp_adver_gif.write(base64.b64decode(gif_str)) tmp_adver_gif.close() elif model == 'mnist': tmp_adver = NamedTemporaryFile() tmp_adver_gif = NamedTemporaryFile() tmp_f = NamedTemporaryFile() mnist_sample = int(request.POST.get("sample", None)) mnist_target = int(request.POST.get("target", None)) mnist_algorithm = request.POST.get("mnist_algorithm", None) sample_result, adver_result, attack_speed = mnist_attack_func( mnist_sample, mnist_target, mnist_algorithm) print("attack speed: %s seconds" % (round(attack_speed, 5))) print('original class:', mnist_sample, 'target class:', mnist_target) print('adversarial example is classified by', np.argmax(adver_result)) sample_result = sample_result.tolist() adver_result = adver_result.tolist() with open( os.path.join(current_dir, 'mnist/dataset/images/testtest.png'), 'rb') as input_file: input_str = base64.b64encode(input_file.read()) tmp_f.write(base64.b64decode(input_str)) tmp_f.close() with open(os.path.join(current_dir, 'mnist/output/testtest.png'), 'rb') as img_file: img_str = base64.b64encode(img_file.read()) tmp_adver.write(base64.b64decode(img_str)) tmp_adver.close() with open(os.path.join(current_dir, 'mnist/output/testtest.gif'), 'rb') as gif_file: gif_str = base64.b64encode(gif_file.read()) tmp_adver_gif.write(base64.b64decode(gif_str)) tmp_adver_gif.close() # Make Graph data["attack_speed"] = attack_speed data["success"] = True data["confidence"] = {} if model == 'imagenet': data["model"] = 'imagenet' for i in range(len(classify_result)): data["confidence"][classify_result[i][1]] = float( classify_result[i][2]) data["adverimage"] = 'data:image/png;base64,' + img_str.decode( 'utf-8') data["adverimage_gif"] = 'data:image/gif;base64,' + gif_str.decode( 'utf-8') data["adversarial"] = {} for i in range(len(result)): data["adversarial"][result[i][1]] = float(result[i][2]) sess.close() elif model == 'mnist': data["model"] = 'mnist' for i in range(len(sample_result[0])): data["confidence"][str(i)] = float(sample_result[0][i]) data["input_image"] = 'data:image/png;base64,' + input_str.decode( 'utf-8') data["adverimage"] = 'data:image/png;base64,' + img_str.decode( 'utf-8') data["adverimage_gif"] = 'data:image/gif;base64,' + gif_str.decode( 'utf-8') data["adversarial"] = {} for i in range(len(adver_result[0])): data["adversarial"][str(i)] = float(adver_result[0][i]) return JsonResponse(data)
def restore_documents( restore_file: io.BytesIO, user: User, skip_ocr=False ): restore_file.seek(0) with tarfile.open(fileobj=restore_file, mode="r") as restore_archive: backup_json = restore_archive.extractfile('backup.json') backup_info = json.load(backup_json) leading_user_in_path = False _user = user if not user: leading_user_in_path = True # user was not specified. It is assument that # backup.json contains a list of users. # Thus recreate users first. for backup_user in backup_info['users']: User.objects.create( username=backup_user['username'], email=backup_user['email'], is_active=backup_user['is_active'], is_superuser=backup_user['is_superuser'] ) for restore_file in restore_archive.getnames(): if restore_file == "backup.json": continue splitted_path = PurePath(restore_file).parts base, ext = os.path.splitext(splitted_path[-1]) # if there is leading username, remove it. if leading_user_in_path: username = splitted_path[0] _user = User.objects.get(username=username) splitted_path = splitted_path[1:] if backup_info.get('documents', False): backup_info_documents = backup_info['documents'] else: backup_info_documents = _get_json_user_documents_list( backup_info, _user ) leading_user_in_path = True for info in backup_info_documents: document_info = info if info['path'] == restore_file: break parent = None # we first have to create a folder structure if len(splitted_path) > 1: for folder in splitted_path[:-1]: folder_object = Folder.objects.filter( title=folder, user=_user ).filter(parent=parent).first() if folder_object is None: new_folder = Folder.objects.create( title=folder, parent=parent, user=_user ) parent = new_folder else: parent = folder_object document_object = Document.objects.filter( title=splitted_path[-1], user=_user ).filter(parent=parent).first() if document_object is not None: logger.error( "Document %s already exists, skipping", restore_file ) else: with NamedTemporaryFile("w+b", suffix=ext) as temp_output: temp_output.write( restore_archive.extractfile(restore_file).read() ) temp_output.seek(0) size = os.path.getsize(temp_output.name) page_count = get_pagecount(temp_output.name) if parent: parent_id = parent.id else: parent_id = None new_doc = Document.create_document( user=_user, title=splitted_path[-1], size=size, lang=document_info['lang'], file_name=splitted_path[-1], parent_id=parent_id, notes="", page_count=page_count) default_storage.copy_doc( src=temp_output.name, dst=new_doc.path.url() ) for page_num in range(1, page_count + 1): if not skip_ocr: ocr_page.apply_async(kwargs={ 'user_id': _user.id, 'document_id': new_doc.id, 'file_name': splitted_path[-1], 'page_num': page_num, 'lang': document_info['lang']} )
def download(self, request): """ Download individual or multiple media assets. Multiple media assets are zipped. """ # get pks pks = get_pks(request.GET) # get list of media media = Media.objects.filter(pk__in=pks).order_by('caption') n_media = media.count() # no media? if n_media == 0: raise Http404( 'Unable to export media assets for empty list of media objects.' ) # single media item? if n_media == 1: item = media.first() response = FileResponse(open(item.original_path, 'rb')) response[ 'Content-Disposition'] = 'attachment; filename="%s"' % item.filename return response # multiple assets -> create zip file f = NamedTemporaryFile() zf = zipfile.ZipFile(f, 'w') # attach original files to zip, handle duplicated filenames filenames = {} for item in media: # determine unique filename filename = item.filename if filename not in filenames: filenames[filename] = 1 else: fn = filename base, ext = os.path.splitext(filename) while fn in filenames: filenames[fn] += 1 fn = '%s-%d%s' % (base, filenames[fn], ext) filename = fn filenames[filename] = 1 # attach file to zip... zf.write(item.original_path, filename) zf.close() f.seek(0) # determine site name from settings (CMS) if 'cubane.cms' in settings.INSTALLED_APPS: from cubane.cms.views import get_cms_settings site_name = get_cms_settings().name if site_name is None: site_name = '' site_name = slugify(site_name) if site_name != '': site_name += '_' else: site_name = '' # generate download filename for zip file today = datetime.date.today() filename = '{0}media_{1:02d}_{2:02d}_{3:04d}.zip'.format( site_name, today.day, today.month, today.year) # serve file response = FileResponse(f) response[ 'Content-Disposition'] = 'attachment; filename="%s"' % filename return response
def input(self, **kwargs): encoding = self.default_encoding options = dict(self.options) if self.infile is None and "{infile}" in self.command: # create temporary input file if needed if self.filename is None: self.infile = NamedTemporaryFile(mode='wb') self.infile.write(self.content.encode(encoding)) self.infile.flush() options["infile"] = self.infile.name else: # we use source file directly, which may be encoded using # something different than utf8. If that's the case file will # be included with charset="something" html attribute and # charset will be available as filter's charset attribute encoding = self.charset # or self.default_encoding self.infile = open(self.filename) options["infile"] = self.filename if "{outfile}" in self.command and "outfile" not in options: # create temporary output file if needed ext = self.type and ".%s" % self.type or "" self.outfile = NamedTemporaryFile(mode='r+', suffix=ext) options["outfile"] = self.outfile.name # Quote infile and outfile for spaces etc. if "infile" in options: options["infile"] = shell_quote(options["infile"]) if "outfile" in options: options["outfile"] = shell_quote(options["outfile"]) try: command = self.command.format(**options) proc = subprocess.Popen(command, shell=True, cwd=self.cwd, stdout=self.stdout, stdin=self.stdin, stderr=self.stderr) if self.infile is None: # if infile is None then send content to process' stdin filtered, err = proc.communicate(self.content.encode(encoding)) else: filtered, err = proc.communicate() filtered, err = filtered.decode(encoding), err.decode(encoding) except (IOError, OSError) as e: raise FilterError('Unable to apply %s (%r): %s' % (self.__class__.__name__, self.command, e)) else: if proc.wait() != 0: # command failed, raise FilterError exception if not err: err = ('Unable to apply %s (%s)' % (self.__class__.__name__, self.command)) if filtered: err += '\n%s' % filtered raise FilterError(err) if self.verbose: self.logger.debug(err) outfile_path = options.get('outfile') if outfile_path: with io.open(outfile_path, 'r', encoding=encoding) as file: filtered = file.read() finally: if self.infile is not None: self.infile.close() if self.outfile is not None: self.outfile.close() return smart_text(filtered)
def test_articolo(self): CONTENUTO_1 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit.\n" NOME_1 = 'Test1.txt' CONTENUTO_2 = "Donec tempus nisi eu enim consequat, non scelerisque nisi accumsan.\n" NOME_2 = 'Test/Test2.txt' volontario, _, _ = crea_persona_sede_appartenenza() presidente = crea_persona() presidente.save() presidente, sede, _ = crea_persona_sede_appartenenza(presidente) delega_presidente_in_corso = Delega( persona=presidente, tipo=PRESIDENTE, oggetto=sede, inizio=datetime.datetime.now() - datetime.timedelta(days=5), fine=datetime.datetime.now() + datetime.timedelta(days=5) ) delega_presidente_in_corso.save() articolo = Articolo.objects.create( titolo='Titolo 1', corpo=parola_casuale(3000), data_inizio_pubblicazione=datetime.datetime.now() - datetime.timedelta(days=5), ) self.assertEqual(articolo.corpo[:Articolo.DIMENSIONE_ESTRATTO], articolo.estratto) self.assertFalse(articolo.termina) articolo2 = Articolo.objects.create( titolo='Titolo 2', corpo='Testo random', estratto='estratto qualsiasi', data_inizio_pubblicazione=datetime.datetime.now() - datetime.timedelta(days=5), data_fine_pubblicazione=datetime.datetime.now() + datetime.timedelta(days=5), stato=Articolo.PUBBLICATO ) segmento_presidenti_no_filtri = ArticoloSegmento.objects.create( segmento='I', articolo=articolo2, ) self.assertNotEqual(articolo2.corpo, articolo2.estratto) self.assertEqual(articolo2.estratto, 'estratto qualsiasi') self.assertTrue(articolo2.termina) articolo3 = Articolo.objects.create( titolo='Titolo 3', corpo='Testo qualsiasi', estratto='estratto random', data_inizio_pubblicazione=datetime.datetime.now() - datetime.timedelta(days=5), stato=Articolo.PUBBLICATO ) segmento_volontari_no_filtri = ArticoloSegmento.objects.create( segmento='B', articolo=articolo3 ) z = Zip(oggetto=articolo3) f1 = NamedTemporaryFile(delete=False, mode='wt') f1.write(CONTENUTO_1) f1.close() z.aggiungi_file(f1.name, NOME_1) z.comprimi_e_salva(nome='TestZip.zip') self.assertEqual(1, articolo3.allegati.all().count()) self.assertIn(z, articolo3.allegati.all()) articolo4 = Articolo.objects.create( titolo='Titolo 4', corpo='Testo qualsiasi 2', estratto='estratto random 2', data_inizio_pubblicazione=datetime.datetime.now() - datetime.timedelta(days=5), data_fine_pubblicazione=datetime.datetime.now() - datetime.timedelta(days=2), stato=Articolo.PUBBLICATO ) pubblicati = Articolo.objects.pubblicati() bozze = Articolo.objects.bozze() self.assertEqual(pubblicati.count(), 2) self.assertEqual(bozze.count(), 1) self.assertIn(articolo, bozze) self.assertNotIn(articolo, pubblicati) self.assertNotIn(articolo2, bozze) self.assertIn(articolo2, pubblicati) self.assertNotIn(articolo3, bozze) self.assertIn(articolo3, pubblicati) self.assertNotIn(articolo4, bozze) self.assertNotIn(articolo4, pubblicati) segmenti_volontario = ArticoloSegmento.objects.all().filtra_per_segmenti(volontario) articoli_volontario = segmenti_volontario.oggetti_collegati() self.assertNotIn(articolo2, articoli_volontario) self.assertIn(articolo3, articoli_volontario) segmenti_presidente = ArticoloSegmento.objects.all().filtra_per_segmenti(presidente) articoli_presidente = segmenti_presidente.oggetti_collegati() self.assertIn(articolo2, articoli_presidente) self.assertIn(articolo3, articoli_presidente)
print "total=%s, count=%s, exist=%s, new=%s" % (total, count, exist, new) continue else: #если отрицательно, создаем чек print 'CREATE CHECK %s' % count, i['file'] new = new+1 c=check.objects.create(fname=i['file']) print "total=%s, count=%s, exist=%s, new=%s" % (total, count, exist, new) # print 'READ FILE %s' % count, i['file'] #читаем файл, перекодируем в utf8 tmp=open('%s/%s' % (i['root'], i['file'])).read().decode('cp1251').encode('utf8') #создаем временный файл f = NamedTemporaryFile(delete=True) f.write(tmp) #пишем открытый файл во временный f.close() #ПИШЕМ ЗАГОЛОВОК ЧЕКА, ТОЛЬКО ПЕРВУЮ СТРОКУ, ОСТАЛЬНЫЕ ДУБЛЬ with open(f.name) as csvfile: reader = csv.DictReader(csvfile, delimiter=';', quotechar='"') for value in reader: print "HEADER %s %s %s" % (value['НомерЧекаККМ'], value['Дата'], value['Цена']) #берем дату getdate=datetime.datetime.strptime(value['Дата'],"%d.%m.%Y %H:%M:%S") #костыли try: #пробуем взять магазины
def get_bootstrapdata(self): url = 'https://www.airbnb.co.kr/s/homes?query=서울특별시§ion_offset=5' headers = { # 'cache-control': "no-cache", # 'user-agent': 'Mozilla/5.0', # 'postman-token': '912622a3-b1b0-49ab-89e9-ae6ab262184f', # 'host': 'www.airbnb.co.kr', # 'accept': '*/*', # 'accept-encoding': 'gzip, deflate', # 'cookie': '__svt=1381; cache_state=0; bev=1523934971_5ussJFyJsy%2BcIwMA; _csrf_token=V4%24.airbnb.co.kr%24G6i45M0XMrs%24reAsEi8IPWdU9YJsn2apAPDEIOtNCuY26kcvWPawU70%3D; jitney_client_session_id=2f0abc70-0be3-4f8d-aac3-84363faa4a73; jitney_client_session_created_at=1523957423; jitney_client_session_updated_at=1523957423; _user_attributes=%7B%22curr%22%3A%22KRW%22%2C%22guest_exchange%22%3A1072.3055650000001%2C%22device_profiling_session_id%22%3A%221523934971--2a4ffad382218d61b920a404%22%2C%22giftcard_profiling_session_id%22%3A%221523957424--c718f2fe50a984671b19354d%22%2C%22reservation_profiling_session_id%22%3A%221523957424--48f5ec024f0477b119d21220%22%7D; flags=268697600; dtc_exp=1; 9f2398a3e=control; b3b78300e=control; 65e98c419=control; a8f0b01c0=control; 4b522145b=treatment', 'cookie': '__svt=1381; cache_state=0; bev=1523934971_5ussJFyJsy%2BcIwMA; _csrf_token=V4%24.airbnb.co.kr%24G6i45M0XMrs%24reAsEi8IPWdU9YJsn2apAPDEIOtNCuY26kcvWPawU70%3D; jitney_client_session_id=7409e864-df24-46e1-9331-5793fc836ac4; jitney_client_session_created_at=1523979136; jitney_client_session_updated_at=1523979136; _user_attributes=%7B%22curr%22%3A%22KRW%22%2C%22guest_exchange%22%3A1072.3055650000001%2C%22device_profiling_session_id%22%3A%221523934971--2a4ffad382218d61b920a404%22%2C%22giftcard_profiling_session_id%22%3A%221523979138--ded77b2c438cd5e5b8f64f9e%22%2C%22reservation_profiling_session_id%22%3A%221523979138--403d68f09bc045f3db938d8a%22%7D; flags=268697600; dtc_exp=1; 9f2398a3e=control; b3b78300e=control; 65e98c419=control; a8f0b01c0=control; 4b522145b=treatment; Cookie_1=value; mdr_browser=desktop', } # 1) Airbnb web crawling # response = requests.get(url) # response = self.r.get(url, headers=headers) # response = requests.get(url, headers=headers) # print(response.status_code) # source = response.text # 2) local airbnb_data.html source = open('airbnb_local_data.html', 'rt', encoding='utf8').read() print(source) bootstrap_data = re.search(r'data-hypernova-key="spaspabundlejs" data-hypernova-id=".*?"><!--(.*?)--></script>', source) # print(bootstrap_data.groups(1)[0:10]) bootstrap_json = json.loads(bootstrap_data.group(1)) # listing 18개가 들어있는 list listing_list = \ bootstrap_json['bootstrapData']['reduxData']['exploreTab']['response']['explore_tabs'][0]['sections'][0][ 'listings'] for i in range(len(listing_list)): print(listing_list[i]['listing']) listing = listing_list[i]['listing'] # crawling data에서 image 다운받기 # img_cover_binary_data = requests.get(listing['picture_url']).content response = requests.get(listing['picture_urls'][0]).content print(type(response)) print(response) print(ContentFile(response)) print(type(ContentFile(response))) # print(ContentFile(requests.get(listing['picture_urls'][2]).content)) temp_file = NamedTemporaryFile(suffix='.png') temp_file.seek(0) temp_file.write(response) temp_file.seek(0) print(temp_file.name) img_cover = open(temp_file.name, 'rb') print(img_cover) print(type(img_cover)) # crawling data 로 house info 채우기 data = { 'house_type': 'HO', 'name': listing['name'], 'description': '테스트용 집입니다.', 'room': listing['bedrooms'], 'bed': listing['beds'], 'bathroom': listing['bathrooms'], 'personnel': listing['person_capacity'], 'amenities': [], 'facilities': [1, 2, 3, 4, 5], 'minimum_check_in_duration': 1, 'maximum_check_in_duration': 3, 'maximum_check_in_range': 90, 'price_per_night': 100000, 'country': 'default', 'city': 'default', 'district': listing['localized_city'], 'dong': 'default', 'address1': 'default', # 'address2': '희망빌라 2동 301호', 'latitude': listing['lat'], 'longitude': listing['lng'], 'disable_days': [ '2014-01-01', '2014-02-01', '2014-03-01', '2014-04-01', ], # 'img_cover': ContentFile(requests.get(listing['picture_urls'][0]).content), 'img_cover': img_cover, # 'house_images': [ # ContentFile(requests.get(listing['picture_urls'][1]).content), # ContentFile(requests.get(listing['picture_urls'][2]).content), # ], } # host_user의 Token 값 header에 넣기 # 회원가입 # 로그인 user_data = { 'username': '******', 'password': '******' } # response = requests.post('/user/login/', user_data) response = requests.post('http://localhost:8000/user/login/', user_data) result = json.loads(response.text) token = result['token'] headers = { 'Authorization': 'Token ' + token, } # response = requests.post('/house/', data, headers=headers) response = requests.post('http://localhost:8000/house/', data, headers=headers) print(response) print(i+1) img_cover.close() temp_file.close()
def generate_export(export_type, xform, export_id=None, options=None, retries=0): """ Create appropriate export object given the export type. param: export_type param: xform params: export_id: ID of export object associated with the request param: options: additional parameters required for the lookup. binary_select_multiples: boolean flag end: end offset ext: export extension type dataview_pk: dataview pk group_delimiter: "/" or "." query: filter_query for custom queries remove_group_name: boolean flag split_select_multiples: boolean flag index_tag: ('[', ']') or ('_', '_') """ username = xform.user.username id_string = xform.id_string end = options.get("end") extension = options.get("extension", export_type) filter_query = options.get("query") remove_group_name = options.get("remove_group_name", False) start = options.get("start") export_type_func_map = { Export.XLS_EXPORT: 'to_xls_export', Export.CSV_EXPORT: 'to_flat_csv_export', Export.CSV_ZIP_EXPORT: 'to_zipped_csv', Export.SAV_ZIP_EXPORT: 'to_zipped_sav', Export.GOOGLE_SHEETS_EXPORT: 'to_google_sheets', } if xform is None: xform = XForm.objects.get( user__username__iexact=username, id_string__iexact=id_string) dataview = None if options.get("dataview_pk"): dataview = DataView.objects.get(pk=options.get("dataview_pk")) records = dataview.query_data(dataview, all_data=True, filter_query=filter_query) total_records = dataview.query_data(dataview, count=True)[0].get('count') else: records = query_data(xform, query=filter_query, start=start, end=end) if filter_query: total_records = query_data(xform, query=filter_query, start=start, end=end, count=True)[0].get('count') else: total_records = xform.num_of_submissions if isinstance(records, QuerySet): records = records.iterator() export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True \ if export_type == Export.SAV_ZIP_EXPORT else remove_group_name export_builder.GROUP_DELIMITER = options.get( "group_delimiter", DEFAULT_GROUP_DELIMITER ) export_builder.SPLIT_SELECT_MULTIPLES = options.get( "split_select_multiples", True ) export_builder.BINARY_SELECT_MULTIPLES = options.get( "binary_select_multiples", False ) export_builder.INCLUDE_LABELS = options.get('include_labels', False) export_builder.INCLUDE_LABELS_ONLY = options.get( 'include_labels_only', False ) export_builder.INCLUDE_HXL = options.get('include_hxl', False) export_builder.INCLUDE_IMAGES \ = options.get("include_images", settings.EXPORT_WITH_IMAGE_DEFAULT) export_builder.VALUE_SELECT_MULTIPLES = options.get( 'value_select_multiples', False) export_builder.REPEAT_INDEX_TAGS = options.get( "repeat_index_tags", DEFAULT_INDEX_TAGS ) # 'win_excel_utf8' is only relevant for CSV exports if 'win_excel_utf8' in options and export_type != Export.CSV_EXPORT: del options['win_excel_utf8'] export_builder.set_survey(xform.survey, xform) temp_file = NamedTemporaryFile(suffix=("." + extension)) columns_with_hxl = export_builder.INCLUDE_HXL and get_columns_with_hxl( xform.survey_elements) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) try: func.__call__( temp_file.name, records, username, id_string, filter_query, start=start, end=end, dataview=dataview, xform=xform, options=options, columns_with_hxl=columns_with_hxl, total_records=total_records ) except NoRecordsFoundError: pass except SPSSIOError as e: export = get_or_create_export(export_id, xform, export_type, options) export.error_message = str(e) export.internal_status = Export.FAILED export.save() report_exception("SAV Export Failure", e, sys.exc_info()) return export # generate filename basename = "%s_%s" % ( id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")) if remove_group_name: # add 'remove group name' flag to filename basename = "{}-{}".format(basename, GROUPNAME_REMOVED_FLAG) if dataview: basename = "{}-{}".format(basename, DATAVIEW_EXPORT) filename = basename + "." + extension # check filename is unique while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) file_path = os.path.join( username, 'exports', id_string, export_type, filename) # TODO: if s3 storage, make private - how will we protect local storage?? # seek to the beginning as required by storage classes temp_file.seek(0) export_filename = default_storage.save(file_path, File(temp_file, file_path)) temp_file.close() dir_name, basename = os.path.split(export_filename) # get or create export object export = get_or_create_export(export_id, xform, export_type, options) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL # do not persist exports that have a filter # Get URL of the exported sheet. if export_type == Export.GOOGLE_SHEETS_EXPORT: export.export_url = export_builder.url # if we should create a new export is true, we should not save it if start is None and end is None: export.save() return export
def generate_export(export_type, extension, username, id_string, export_id=None, filter_query=None, group_delimiter='/', split_select_multiples=True, binary_select_multiples=False, sync_to_gsuit=False, user=None): """ Create appropriate export object given the export type """ time.sleep(5) export_type_func_map = { Export.XLS_EXPORT: 'to_xls_export', Export.CSV_EXPORT: 'to_flat_csv_export', Export.CSV_ZIP_EXPORT: 'to_zipped_csv', Export.SAV_ZIP_EXPORT: 'to_zipped_sav', Export.ANALYSER_EXPORT: 'to_analyser_export' } xform = XForm.objects.get(user__username__iexact=username, id_string__exact=id_string) # query mongo for the cursor records = query_mongo(username, id_string, filter_query) export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = group_delimiter export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples __version__ = "0" try: __version__ = filter_query['$and'][0]['__version__'] except Exception as e: print(str(e)) if __version__: survey = build_survey_from_history(xform, __version__) if not survey: export_builder.set_survey(xform.data_dictionary().survey) else: export_builder.set_survey(survey) else: export_builder.set_survey(xform.data_dictionary().survey) prefix = slugify('{}_export__{}__{}'.format(export_type, username, id_string)) temp_file = NamedTemporaryFile(prefix=prefix, suffix=("." + extension)) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) func.__call__(temp_file.name, records, username, id_string, filter_query) # generate filename basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) if export_type == Export.ANALYSER_EXPORT: # Analyser exports should be distinguished by more than just their file extension. basename = '{}_ANALYSER_{}'.format( id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + extension # check filename is unique while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) file_path = os.path.join(username, 'exports', id_string, export_type, filename) # TODO: if s3 storage, make private - how will we protect local storage?? storage = get_storage_class()() # seek to the beginning as required by storage classes print sync_to_gsuit, 'file_url--------->', temp_file, filter_query try: if sync_to_gsuit == True and '__version__' not in filter_query['$and'][ 0]: if not os.path.exists("media/forms/"): os.makedirs("media/forms/") temporarylocation = "media/forms/submissions_{}.xls".format( id_string) import shutil shutil.copy(temp_file.name, temporarylocation) fxf_form = FieldSightXF.objects.get( pk=filter_query['$and'][0]['fs_project_uuid']) upload_to_drive(temporarylocation, str(fxf_form.id) + '_' + id_string, None, fxf_form.project, user) os.remove(temporarylocation) except Exception as e: print e.__dict__ # get or create export object temp_file.seek(0) export_filename = storage.save(file_path, File(temp_file, file_path)) dir_name, basename = os.path.split(export_filename) temp_file.close() if export_id: export = Export.objects.get(id=export_id) else: fsxf = filter_query.values()[0] # print("fsxf", fsxf) export = Export(xform=xform, export_type=export_type, fsxf_id=fsxf) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL # dont persist exports that have a filter if filter_query is None: export.save() export.save() return export
def test_file_truncates_namedtemporaryfile_name(self): named_file = NamedTemporaryFile() f = File(named_file) self.assertEqual(f.name, os.path.basename(named_file.name))
"error": "We failed to reach a server, reason: %s" % e.reason }, context_instance=RequestContext(request)) elif hasattr(e, "code"): return render_to_response("error.html", { "error": "The remote server couldn't fulfill the request, HTTP error code %s" % e.code }, context_instance=RequestContext( request)) # Store temp file. url_temp = NamedTemporaryFile(delete=True) url_temp.write(url.read()) url_temp.flush() # Convert to File object. url_file = File(url_temp).name # Check content type. mime = magic.Magic(mime=True) content_type = mime.from_file(url_file) if not check_allowed_content(content_type): return render_to_response( "error.html", {"error": "File type not supported"}, context_instance=RequestContext(request)) # Create analysis task.
def dataToImageFile(data): image = NamedTemporaryFile(delete=False) image.write(data) image.flush() return ImageFile(image)
def export_handler(request, tag=None, package_id=None, branch=None, version_guid=None, block=None): """ The restful handler for exporting a course. GET html: return html page for import page application/x-tgz: return tar.gz file containing exported course json: not supported Note that there are 2 ways to request the tar.gz file. The request header can specify application/x-tgz via HTTP_ACCEPT, or a query parameter can be used (?_accept=application/x-tgz). If the tar.gz file has been requested but the export operation fails, an HTML page will be returned which describes the error. """ location = BlockUsageLocator(package_id=package_id, branch=branch, version_guid=version_guid, block_id=block) if not has_course_access(request.user, location): raise PermissionDenied() old_location = loc_mapper().translate_locator_to_location(location) course_module = modulestore().get_item(old_location) # an _accept URL parameter will be preferred over HTTP_ACCEPT in the header. requested_format = request.REQUEST.get( '_accept', request.META.get('HTTP_ACCEPT', 'text/html')) export_url = location.url_reverse('export') + '?_accept=application/x-tgz' if 'application/x-tgz' in requested_format: name = old_location.name export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz") root_dir = path(mkdtemp()) try: export_to_xml(modulestore('direct'), contentstore(), old_location, root_dir, name, modulestore()) except SerializationError, e: logging.exception( 'There was an error exporting course {0}. {1}'.format( course_module.location, unicode(e))) unit = None failed_item = None parent = None try: failed_item = modulestore().get_instance( course_module.location.course_id, e.location) parent_locs = modulestore().get_parent_locations( failed_item.location, course_module.location.course_id) if len(parent_locs) > 0: parent = modulestore().get_item(parent_locs[0]) if parent.location.category == 'vertical': unit = parent except: # if we have a nested exception, then we'll show the more generic error message pass unit_locator = loc_mapper().translate_location( old_location.course_id, parent.location, False, True) return render_to_response( 'export.html', { 'context_course': course_module, 'in_err': True, 'raw_err_msg': str(e), 'failed_module': failed_item, 'unit': unit, 'edit_unit_url': unit_locator.url_reverse("unit") if parent else "", 'course_home_url': location.url_reverse("course"), 'export_url': export_url }) except Exception, e: logging.exception( 'There was an error exporting course {0}. {1}'.format( course_module.location, unicode(e))) return render_to_response( 'export.html', { 'context_course': course_module, 'in_err': True, 'unit': None, 'raw_err_msg': str(e), 'course_home_url': location.url_reverse("course"), 'export_url': export_url })