def testMergeFiles(self): """Test merging multiple files.""" input_data = [(str(i), "_" + str(i)) for i in range(100)] input_data.sort() bucket_name = "testbucket" test_filename = "testfile" full_filename = "/%s/%s" % (bucket_name, test_filename) with cloudstorage.open(full_filename, mode="w") as f: with records.RecordsWriter(f) as w: for (k, v) in input_data: proto = kv_pb.KeyValue() proto.set_key(k) proto.set_value(v) w.write(proto.Encode()) p = TestMergePipeline(bucket_name, [full_filename, full_filename, full_filename]) p.start() test_support.execute_until_empty(self.taskqueue) p = TestMergePipeline.from_id(p.pipeline_id) output_file = p.outputs.default.value[0] output_data = [] with cloudstorage.open(output_file) as f: for record in records.RecordsReader(f): output_data.append(record) expected_data = [str((k, [v, v, v], False)) for (k, v) in input_data] self.assertEquals(expected_data, output_data) self.assertEquals(1, len(self.emails))
def process_images(): if request.method == 'POST': currentFileName = request.form['filename'] newFormat = request.form['format'] encodingParameter = images.JPEG if newFormat == 'png': encodingParameter = images.PNG if newFormat == 'gif': encodingParameter = images.GIF if newFormat == 'bmp': encodingParameter = images.BMP newFileName = "" for i in range(len(currentFileName.split('.'))-1 ): newFileName = newFileName + currentFileName.split('.')[i] + '.' newFileName = newFileName + newFormat f = gcs.open( BUCKET_PATH + currentFileName) img = images.Image(image_data=f.read()) f.close() try: newWidth = int(request.form['width']) except: newWidth = img.height try: newHeight = int(request.form['height']) except: newHeight = img.width img.resize(width= newWidth, height=newHeight, allow_stretch=True ) processedImage = img.execute_transforms(output_encoding=encodingParameter) f = gcs.open(BUCKET_PATH + newFileName, "w") f.write(processedImage) f.close() return final_upload(newFileName)
def __getBlob(cls, _key): """ Private method that gets blob content param @key is String return tuple (value, tll) """ if not _key is None: _clau = cls.__getBlobkey(_key) if not _clau is None: br = gcs.open(cls.__bucket_name + _key) value = br.read() br.close() return cls.__checkIfExpired(value); else: z=0 _clau = cls.__getBlobkey(_key+"_"+str(z)) s=[] while not _clau is None: br = gcs.open(cls.__bucket_name + _clau) s.append(br.read()) br.close() z=z+1 _clau = cls.__getBlobkey(_key+"_"+str(z)) if z>0: value = "".join(s) return cls.__checkIfExpired(value); else: return (None,None) else: return (None,None)
def run(self): #INSERT OPERATION (in cache) for name in self.filenames: if((filesizes[name]<=100000) and self.use_cache): #If the size of the file is < 100 kb and memcache option is on name_gcs = bucket+'/'+ name.encode() gcs_file = gcs.open(name_gcs, 'r') value = gcs_file.read() insertCache(name, value) #RETRIEVE OPERATION start_retrieve = datetime.now() fileaccess = numpy.random.random_integers(0,len(self.filenames)-1,2*len(self.filenames)) #Generating 2 random accesses per file following uniform distribution for i in range(len(fileaccess)): key = self.filenames[fileaccess[i]] filename = bucket + '/' + key.encode() value = findCache(key) if (value is None): if check(filename): with gcs.open(filename, 'r') as f: for line in f: value = f.readline() RETRIEVE_TIME.append((datetime.now() - start_retrieve).total_seconds()) #REMOVE OPERATION start_remove = datetime.now() for name in self.filenames: remove(name) REMOVE_TIME.append((datetime.now() - start_remove).total_seconds())
def Check(EmployeeDataFilePath,emailID,fallOutReport): FileName = EmployeeDataFilePath.split('/')[-1].split(".xlsx")[0] EmployeeDataFilePathTsv = XlsxToTsv(EmployeeDataFilePath) FieldId,Employees,TotalEmployee = EmployeeData(EmployeeDataFilePathTsv) Errors = ErrorList(Employees,TotalEmployee,ProductionTemplateFileName) if len(Errors) == 0 : if fallOutReport == 1: FileName = FallOutReportXlsx(FieldId,Employees,TotalEmployee,ProductionTemplateFileName,FileName) Message = open("SuccessEmailBody.txt").read() Subject = "Success Factor Upload FallOut Report" mail.send_mail(sender=sender_email_id.format( app_identity.get_application_id()), to=emailID, subject=Subject, body=Message,attachments=[(FileName, gcs.open(FileName).read())]) gcs.delete(FileName) else: FileName = XlsxErrorReport(Errors,FileName) Message = open("ErrorEmailBody.txt").read() Subject = "Success Factor Upload File Error" mail.send_mail(sender=sender_email_id.format( app_identity.get_application_id()), to=emailID, subject=Subject, body=Message,attachments=[(FileName, gcs.open(FileName).read())]) gcs.delete(FileName) if len(Errors) == 0: return 1 else: return 0
def testSortFile(self): """Test sorting a file.""" bucket_name = "testbucket" test_filename = "testfile" full_filename = "/%s/%s" % (bucket_name, test_filename) input_data = [(str(i), "_" + str(i)) for i in range(100)] with cloudstorage.open(full_filename, mode="w") as f: with records.RecordsWriter(f) as w: for (k, v) in input_data: proto = kv_pb.KeyValue() proto.set_key(k) proto.set_value(v) w.write(proto.Encode()) p = shuffler._SortChunksPipeline("testjob", bucket_name, [[full_filename]]) p.start() test_support.execute_until_empty(self.taskqueue) p = shuffler._SortChunksPipeline.from_id(p.pipeline_id) input_data.sort() output_files = p.outputs.default.value[0] output_data = [] for output_file in output_files: with cloudstorage.open(output_file) as f: for binary_record in records.RecordsReader(f): proto = kv_pb.KeyValue() proto.ParseFromString(binary_record) output_data.append((proto.key(), proto.value())) self.assertEquals(input_data, output_data) self.assertEquals(1, len(self.emails))
def maybe_process_image(image_url, base_name): if CLOUD_STORAGE_ROOT_URL in image_url: return (image_url, None) image_result = urlfetch.fetch(image_url) if image_result.status_code < 200 or image_result.status_code >= 300: raise IOError('Error downloading image: HTTP %d.' % image_result.status_code) base_filename = re.sub(r'[^\w]+', '-', base_name.strip().lower()) # main image image_gcs_path = '/muzeifeaturedart/' + base_filename + '.jpg' # upload with default ACLs set on the bucket # or use options={'x-goog-acl': 'public-read'}) gcs_file = gcs.open(image_gcs_path, 'w', content_type='image/jpeg') gcs_file.write(image_result.content) gcs_file.close() # thumb thumb_gcs_path = '/muzeifeaturedart/' + base_filename + '_thumb.jpg' thumb = images.Image(image_result.content) thumb.resize(width=(thumb.width * 600 / thumb.height), height=600) thumb_contents = thumb.execute_transforms(output_encoding=images.JPEG, quality=40) gcs_file = gcs.open(thumb_gcs_path, 'w', content_type='image/jpeg') gcs_file.write(thumb_contents) gcs_file.close() return (CLOUD_STORAGE_ROOT_URL + image_gcs_path, CLOUD_STORAGE_ROOT_URL + thumb_gcs_path)
def finish(self): """ Called when all shards have finished processing """ if self.get_meta().generate_error_csv: self.error_csv_filename = self._error_csv_filename() with cloudstorage.open(self.error_csv_filename, 'w') as f: # Concat all error csvs from shards into 1 file has_written = False for shard in ImportShard.objects.filter(task_id=self.pk, task_model_path=self.model_path): if not shard.error_csv_filename: continue # If this is the first row, write the column headers if not has_written: data = json.loads(shard.source_data_json)[0] cols = getattr(self, "detected_columns", sorted(data.keys())) + [ "errors" ] csvwriter = csv.writer(f) csvwriter.writerow(cols) has_written = True # Write the shard's error file into the master file f.write(cloudstorage.open(shard.error_csv_filename).read()) cloudstorage.delete(shard.error_csv_filename) if has_written: # Create a blobstore key for the GCS file blob_key = create_gs_key('/gs%s' % self.error_csv_filename) self.error_csv = '%s/errors.csv' % blob_key self.save() else: cloudstorage.delete(self.error_csv_filename)
def post(self): # get args self.start_cursor = self.request.get('cursor') self.filtering_event_key = self.request.get('event') self.filename = self.request.get('filename') self.csv_header = self.request.get('csv_header') self.worker_url = self.request.get('worker_url') self.event = Event.get(self.filtering_event_key) if self.filtering_event_key else None # get (base) query, skip query to cursor, filter for sites query = self.get_base_query() if self.start_cursor: query.with_cursor(self.start_cursor) fetched_sites = query.fetch(limit=self.sites_per_task) sites = self.filter_sites(fetched_sites) # write part of csv file to GCS csv_part_gcs_fd = cloudstorage.open( BUCKET_NAME + '/' + self.filename + '.part.' + self.start_cursor, 'w', content_type='text/csv' ) self._write_csv_rows(csv_part_gcs_fd, sites) csv_part_gcs_fd.close() # decide what to do next self.end_cursor = query.cursor() if self.end_cursor and self.start_cursor != self.end_cursor: # chain to next task taskqueue.add( url=self.worker_url, params=self.get_continuation_param_dict(), retry_options=taskqueue.TaskRetryOptions(task_retry_limit=3), ) else: # finish file: combine parts and deduplicate lines logging.info(u"Deduplicating to create %s ..." % self.filename) sio = StringIO() path_prefix = BUCKET_NAME + '/' + self.filename + '.part' for gcs_file_stat in cloudstorage.listbucket(path_prefix): csv_part_gcs_fd = cloudstorage.open(gcs_file_stat.filename) for line in csv_part_gcs_fd: sio.write(line) csv_part_gcs_fd.close() sio.seek(0) deduplicated_lines = set(line for line in sio) # write csv header and deduplicated lines to new file csv_complete_gcs_fd = cloudstorage.open( BUCKET_NAME + '/' + self.filename, 'w', content_type='text/csv' ) csv_complete_gcs_fd.write(self.csv_header.encode('utf-8')) for line in deduplicated_lines: csv_complete_gcs_fd.write(line) csv_complete_gcs_fd.close()
def get(self, event_key): event = Event.get_by_id(event_key) event.prepAwardsMatchesTeams() if event.awards: with cloudstorage.open( self.AWARDS_FILENAME_PATTERN.format(event.year, event_key, event_key), "w" ) as awards_file: writer = csv.writer(awards_file, delimiter=",") for award in event.awards: for recipient in award.recipient_list: team = recipient["team_number"] if type(team) == int: team = "frc{}".format(team) self._writerow_unicode(writer, [award.key.id(), award.name_str, team, recipient["awardee"]]) if event.matches: with cloudstorage.open( self.MATCHES_FILENAME_PATTERN.format(event.year, event_key, event_key), "w" ) as matches_file: writer = csv.writer(matches_file, delimiter=",") for match in event.matches: red_score = match.alliances["red"]["score"] blue_score = match.alliances["blue"]["score"] self._writerow_unicode( writer, [match.key.id()] + match.alliances["red"]["teams"] + match.alliances["blue"]["teams"] + [red_score, blue_score], ) if event.teams: with cloudstorage.open( self.TEAMS_FILENAME_PATTERN.format(event.year, event_key, event_key), "w" ) as teams_file: writer = csv.writer(teams_file, delimiter=",") self._writerow_unicode(writer, [team.key.id() for team in event.teams]) if event.rankings: with cloudstorage.open( self.RANKINGS_FILENAME_PATTERN.format(event.year, event_key, event_key), "w" ) as rankings_file: writer = csv.writer(rankings_file, delimiter=",") for row in event.rankings: self._writerow_unicode(writer, row) if event.alliance_selections: with cloudstorage.open( self.ALLIANCES_FILENAME_PATTERN.format(event.year, event_key, event_key), "w" ) as alliances_file: writer = csv.writer(alliances_file, delimiter=",") for alliance in event.alliance_selections: self._writerow_unicode(writer, alliance["picks"]) self.response.out.write("Done backing up {}!".format(event_key))
def testReadEmptyFile(self): f = cloudstorage.open(TESTFILE, 'w') f.write('') f.close() f = cloudstorage.open(TESTFILE) self.assertEqual('', f.read()) self.assertEqual('', f.read()) f.close()
def testFilenameEscaping(self): name = BUCKET + '/a b/c d/*%$' with cloudstorage.open(name, 'w') as f: f.write('foo') with cloudstorage.open(name) as f: self.assertEqual('foo', f.read()) self.assertEqual(name, cloudstorage.stat(name).filename) bucket = cloudstorage.listbucket(BUCKET) for stat in bucket: self.assertEqual(name, stat.filename) cloudstorage.delete(name)
def testReadSmall(self): f = cloudstorage.open(TESTFILE, 'w') f.write('abcdefghij') f.close() f = cloudstorage.open(TESTFILE, read_buffer_size=3) self.assertEqual('ab', f.read(2)) self.assertEqual('c', f.read(1)) self.assertEqual('de', f.read(2)) self.assertEqual('fghij', f.read()) f.close()
def WriteInBlockSizeTest(self): f = cloudstorage.open(TESTFILE, 'w') f.write('a'*256*1024) f.write('b'*256*1024) f.close() f = cloudstorage.open(TESTFILE) self.assertEqual('a'*256*1024 + 'b'*256*1024, f.read()) self.assertEqual('', f.read()) self.assertEqual('', f.readline()) f.close()
def testFlush2(self): blocksize = 0 with cloudstorage.open(TESTFILE, 'w') as f: blocksize = f._blocksize f.write('a'*(blocksize+1)) f.write('a') f.write('a'*(blocksize-1)) f.flush() self.assertEqual(1, f._buffered) with cloudstorage.open(TESTFILE) as f: self.assertEqual(blocksize*2+1, len(f.read()))
def testReadEntireFile(self): f = cloudstorage.open(TESTFILE, 'w') f.write('abcde') f.close() f = cloudstorage.open(TESTFILE, read_buffer_size=1) self.assertEqual('abcde', f.read()) f.close() f = cloudstorage.open(TESTFILE) self.assertEqual('abcde', f.read(8)) f.close()
def _create_zip(result_path, human_user_email, app_id, data_export_email): zip_path = os.path.join(result_path, 'result.zip') with cloudstorage.open(zip_path, 'w') as zip_stream: with ZipFile(zip_stream, 'w', allowZip64=True) as bzf: for f in cloudstorage.listbucket(os.path.join(result_path, 'result')): with cloudstorage.open(f.filename, 'r') as file_stream: filename = f.filename.replace('%s/result/' % result_path, '') for chunk in file_stream: bzf.writestr(filename, chunk) download_url = get_serving_url(zip_path) deferred.defer(_send_export_email, result_path, human_user_email, app_id, data_export_email, download_url, _queue=DATA_EXPORT_QUEUE)
def finish(self, reportDone=True): """Called when the worker has finished, to allow for any final work to be done.""" progress = None if reportDone: if self.report.ftype == REPORT.XLS: self.gcs_file.close() readable_gcs_file = gcs.open(self.gcs_file.name, 'r') data = readable_gcs_file.read().split("\n") readable_gcs_file.close() self.gcs_file = gcs.open(self.gcs_file.name, 'w') y = 0 for r in data: if not r: continue if y > REPORT.XLS_ROW_LIMIT: logging.warning("Excel report exceeded row limit and was truncated") break y += 1 row = [] try: row = json.loads(r) except Exception, ex: logging.error("Unable to json load row: %s (%s)" % (r, ex)) else: for x, cell in enumerate(row): if cell: if x in self.report.date_columns: self.ws.write(y, x, cell, self.xls_styles['datetime']) else: self.ws.write(y, x, cell) if self.make_sub_reports: #TODO: Write section_work_sheet, survey to excel is not enabled for now though pass self.wb.save(self.gcs_file) self.gcs_file.close() if self.has_section_files(): for section_gcs_file in self.section_gcs_files: section_gcs_file.close() self.report.status = REPORT.DONE self.report.dt_generated = datetime.now() self.report.put() duration = self.report.getDuration() logging.debug("GCSReportWorker finished. Counters: %s. Report ran for %d seconds." % (self.counters, duration)) progress = { "status": REPORT.DONE, "resource":self.report.getGCSFile(), "generated": tools.unixtime(dt=self.report.dt_generated), "report": self.report.json(), "duration": duration }
def finish(self): """ Called when all shards have finished processing """ # Refresh object self = self._meta.model.objects.get(pk=self.pk) # If this was called before, don't do anything if self.status == ImportStatus.FINISHED: return if self.get_meta().generate_error_csv: self.error_csv_filename = self._error_csv_filename() with cloudstorage.open(self.error_csv_filename, 'w') as f: # Concat all error csvs from shards into 1 file has_written = False shards = self.get_shard_model().objects.filter(task_id=self.pk, task_model_path=self.model_path) # The shards haven't necessarily finished writing their error files when this is called, # because that happens in a defer. So we redefer this until they're all done. if [shard for shard in shards if not shard.error_csv_written]: self.defer(self.finish) return for shard in shards: if not shard.error_csv_filename: continue # If this is the first row, write the column headers if not has_written: data = json.loads(shard.source_data_json)[0] cols = getattr(self, "detected_columns", data.keys()) + [ "errors" ] csvwriter = csv.writer(f) csvwriter.writerow(cols) has_written = True # Write the shard's error file into the master file f.write(cloudstorage.open(shard.error_csv_filename).read()) cloudstorage.delete(shard.error_csv_filename) if has_written: # Create a blobstore key for the GCS file blob_key = create_gs_key('/gs%s' % self.error_csv_filename) self.error_csv = '%s/errors.csv' % blob_key else: cloudstorage.delete(self.error_csv_filename) self.status = ImportStatus.FINISHED self.save()
def maybe_process_image(image_url, crop_tuple, base_name): if CLOUD_STORAGE_ROOT_URL in image_url and crop_tuple == NO_CROP_TUPLE: return (image_url, None) image_result = urlfetch.fetch(image_url) if image_result.status_code < 200 or image_result.status_code >= 300: raise IOError('Error downloading image: HTTP %d.' % image_result.status_code) filename = re.sub(r'[^\w]+', '-', base_name.strip().lower()) + '.jpg' # main image image_gcs_path = CLOUD_STORAGE_BASE_PATH + '/fullres/' + filename # resize to max width 4000 or max height 2000 image_contents = image_result.content image = images.Image(image_contents) edited = False if image.height > 2000: image.resize(width=(image.width * 2000 / image.height), height=2000) edited = True elif image.width > 4000: image.resize(width=4000, height=(image.height * 4000 / image.width)) edited = True if crop_tuple != NO_CROP_TUPLE: image.crop(*crop_tuple) edited = True if edited: image_contents = image.execute_transforms(output_encoding=images.JPEG, quality=80) # upload with default ACLs set on the bucket # or use options={'x-goog-acl': 'public-read'}) gcs_file = gcs.open(image_gcs_path, 'w', content_type='image/jpeg') gcs_file.write(image_contents) gcs_file.close() # thumb thumb_gcs_path = CLOUD_STORAGE_BASE_PATH + '/thumbs/' + filename thumb = images.Image(image_result.content) thumb.resize(width=(thumb.width * THUMB_HEIGHT / thumb.height), height=THUMB_HEIGHT) if crop_tuple != NO_CROP_TUPLE: thumb.crop(*crop_tuple) edited = True thumb_contents = thumb.execute_transforms(output_encoding=images.JPEG, quality=40) gcs_file = gcs.open(thumb_gcs_path, 'w', content_type='image/jpeg') gcs_file.write(thumb_contents) gcs_file.close() return (CLOUD_STORAGE_ROOT_URL + image_gcs_path, CLOUD_STORAGE_ROOT_URL + thumb_gcs_path)
def testRemoveGarbage(self): """Make sure abandoned files get removed.""" writer_spec = {self.WRITER_CLS.BUCKET_NAME_PARAM: "unused", self.WRITER_CLS.TMP_BUCKET_NAME_PARAM: "test"} mapreduce_state = self.create_mapreduce_state(output_params=writer_spec) shard_state = self.create_shard_state(1) ctx = context.Context(mapreduce_state.mapreduce_spec, shard_state) context.Context._set(ctx) writer = self.WRITER_CLS.create(mapreduce_state.mapreduce_spec, shard_state.shard_number, 0) writer.begin_slice(None) # our shard our_file = "/test/gae_mr_tmp/DummyMapReduceJobId-tmp-1-very-random" f = cloudstorage.open(our_file, "w") f.write("foo?") f.close() # not our shard their_file = "/test/gae_mr_tmp/DummyMapReduceJobId-tmp-3-very-random" f = cloudstorage.open(their_file, "w") f.write("bar?") f.close() # unrelated file real_file = "/test/this_things_should_survive" f = cloudstorage.open(real_file, "w") f.write("yes, foobar!") f.close() # Make sure bogus file still exists names = [l.filename for l in cloudstorage.listbucket("/test")] self.assertTrue(our_file in names) self.assertTrue(their_file in names) self.assertTrue(real_file in names) # slice end should clean up the garbage writer = self._serialize_and_deserialize(writer) names = [l.filename for l in cloudstorage.listbucket("/test")] self.assertFalse(our_file in names) self.assertTrue(their_file in names) self.assertTrue(real_file in names) # finalize shouldn't change anything writer.finalize(ctx, shard_state) self.assertFalse(our_file in names) self.assertTrue(their_file in names) self.assertTrue(real_file in names)
def testSeekAndTell(self): f = cloudstorage.open(TESTFILE, 'w') f.write('abcdefghij') f.close() f = cloudstorage.open(TESTFILE) f.seek(5) self.assertEqual(5, f.tell()) self.assertEqual('f', f.read(1)) self.assertEqual(6, f.tell()) f.seek(-1, os.SEEK_CUR) self.assertEqual('f', f.read(1)) f.seek(-1, os.SEEK_END) self.assertEqual('j', f.read(1))
def addtoDB(index, url): import json file = gcs.open(BUCKET_PATH + 'db.json', 'r') data = file.read() file.close() db = json.loads(data) if index not in db: db[index] = [url] else: db[index].append(url) json_str = json.dumps(db) file = gcs.open(BUCKET_PATH+'db.json', 'w') file.write(json_str) file.close()
def testPartialRecords(self): """Test merging into partial key values.""" try: self._prev_max_values_count = shuffler._MergePipeline._MAX_VALUES_COUNT # force max values count to extremely low value. shuffler._MergePipeline._MAX_VALUES_COUNT = 1 input_data = [("1", "a"), ("2", "b"), ("3", "c")] input_data.sort() bucket_name = "testbucket" test_filename = "testfile" full_filename = "/%s/%s" % (bucket_name, test_filename) with cloudstorage.open(full_filename, mode="w") as f: with records.RecordsWriter(f) as w: for (k, v) in input_data: proto = file_service_pb.KeyValue() proto.set_key(k) proto.set_value(v) w.write(proto.Encode()) p = TestMergePipeline(bucket_name, [full_filename, full_filename, full_filename]) p.start() test_support.execute_until_empty(self.taskqueue) p = TestMergePipeline.from_id(p.pipeline_id) output_file = p.outputs.default.value[0] output_data = [] with cloudstorage.open(output_file) as f: for record in records.RecordsReader(f): output_data.append(record) expected_data = [ ("1", ["a"], True), ("1", ["a"], True), ("1", ["a"], False), ("2", ["b"], True), ("2", ["b"], True), ("2", ["b"], False), ("3", ["c"], True), ("3", ["c"], True), ("3", ["c"], False), ] self.assertEquals([str(e) for e in expected_data], output_data) finally: shuffler._MergePipeline._MAX_VALUES_COUNT = self._prev_max_values_count self.assertEquals(1, len(self.emails))
def XlsxToTsv(FilePath): reload(sys) sys.setdefaultencoding('utf-8') wb = xlrd.open_workbook(file_contents = gcs.open(FilePath).read()) sh = wb.sheet_by_index(0) FilePathTsv= FilePath.split(".xlsx")[0] csvFile = gcs.open(FilePathTsv+'.tsv', 'w') wr = csv.writer(csvFile,delimiter='\t') for rownum in xrange(sh.nrows): wr.writerow(sh.row_values(rownum)) csvFile.close() gcs.delete(FilePath) return FilePathTsv+".tsv"
def testFlush(self): blocksize = 0 with cloudstorage.open(TESTFILE, 'w') as f: blocksize = f._blocksize f.write('a'*(blocksize-2)) f.write('a'*3) f.write('a') f.flush() self.assertEqual(2, f._buffered) f.flush() f.write('a') f.close() with cloudstorage.open(TESTFILE) as f: self.assertEqual(blocksize + 3, len(f.read()))
def setUpMultipleFile(self): f = cloudstorage.open(self.seg_prefix + "0", "w", options={self.writer_cls._VALID_LENGTH: "5"}) f.write("12345garbage") f.close() f = cloudstorage.open(self.seg_prefix + "1", "w", options={self.writer_cls._VALID_LENGTH: "5"}) f.write("67890garbage") f.close() f = cloudstorage.open(self.seg_prefix + "2", "w", options={self.writer_cls._VALID_LENGTH: "6"}) f.write("123456garbage") f.close()
def _shrink_note(self, note): for file_key in note.files: file = file_key.get() try: with cloudstorage.open(file.full_path) as f: image = images.Image(f.read()) image.resize(640) new_image_data = image.execute_transforms() content_t = images_formats.get(str(image.format)) with cloudstorage.open(file.full_path, 'w', content_type=content_t) as f: f.write(new_image_data) except images.NotImageError: pass
def _create_note(self, user, title, content, attachments): note = Note(parent=ndb.Key("User", user.nickname()), title=title, content=content) note.put() if attachments: bucket_name = app_identity.get_default_gcs_bucket_name() for file_name, file_content in attachments: content_t = mimetypes.guess_type(file_name)[0] real_path = os.path.join('/', bucket_name, user.user_id(), file_name) with cloudstorage.open(real_path, 'w', content_type=content_t, options={'x-goog-acl': 'public-read'}) as f: f.write(file_content.decode()) key = blobstore.create_gs_key('/gs' + real_path) try: url = images.get_serving_url(key, size=0) thumbnail_url = images.get_serving_url(key, size=150, crop=True) except images.TransformationError, images.NotImageError: url = "http://storage.googleapis.com{}".format(real_path) thumbnail_url = None f = NoteFile(parent=note.key, name=file_name, url=url, thumbnail_url=thumbnail_url, full_path=real_path) f.put() note.files.append(f.key) note.put()
def get_image(image_key): """Serve the content image with the given key.""" import cloudstorage as gcs if not image_key: logging.error("No image key provided") return image = memcache.get(image_key) if image: return Response(image, mimetype='image/jpeg') image = gcs.open('/content_images/%s' % image_key) data = image.read() response = Response(data, mimetype='image/jpeg') image.close() # TODO: Memcache cannot handle images greater than 1MB memcache.set(image_key, data) return response
def set_last_end_time(bucket_name, end_time_str): """ Write the end_time as a string value in a JSON object in GCS. This file is used to remember the last end_time in case one isn't provided """ # get the datetime object end_time = datetime.strptime(end_time_str, '%Y-%m-%dT%H:%M:%S.%fZ') delta = timedelta(seconds=1) # Add 1 second & convert back to str end_time_calc = end_time + delta end_time_calc_str = end_time_calc.strftime('%Y-%m-%dT%H:%M:%S.%fZ') logging.debug("end_time_str: {}, end_time_Calc_str: {}".format( end_time_str, end_time_calc_str)) end_time_str_json = {"end_time": end_time_calc_str} write_retry_params = gcs.RetryParams(backoff_factor=1.1) gcs_file = gcs.open('/{}/{}'.format(bucket_name, config.LAST_END_TIME_FILENAME), 'w', content_type='text/plain', retry_params=write_retry_params) gcs_file.write(json.dumps(end_time_str_json)) gcs_file.close()
def GetTracePaths(bucket): """Returns a list of trace files in a bucket. Finds and loads the trace databases, and returns their content as a list of paths. This function assumes a specific structure for the files in the bucket. These assumptions must match the behavior of the backend: - The trace databases are located in the bucket. - The trace databases files are the only objects with the TRACE_DATABASE_PREFIX prefix in their name. Returns: list: The list of paths to traces, as strings. """ traces = [] prefix = os.path.join('/', bucket, common.clovis_paths.TRACE_DATABASE_PREFIX) file_stats = cloudstorage.listbucket(prefix) for file_stat in file_stats: database_file = file_stat.filename clovis_logger.info('Loading trace database: ' + database_file) with cloudstorage.open(database_file) as remote_file: json_string = remote_file.read() if not json_string: clovis_logger.warning('Failed to download: ' + database_file) continue database = LoadingTraceDatabase.FromJsonString(json_string) if not database: clovis_logger.warning('Failed to parse: ' + database_file) continue for path in database.ToJsonDict(): traces.append(path) return traces
def store_feed_for_band_key(the_band_key, the_feed): bucket_name = os.environ.get('BUCKET_NAME', app_identity.get_default_gcs_bucket_name()) # self.response.headers['Content-Type'] = 'text/plain' # self.response.write('Demo GCS Application running from Version: ' # + os.environ['CURRENT_VERSION_ID'] + '\n') # self.response.write('Using bucket name: ' + bucket_name + '\n\n') filename = "/{0}/{1}{2}".format(bucket_name, "rss-", the_band_key.urlsafe()) # self.response.write('Creating file %s\n' % filename) write_retry_params = gcs.RetryParams(backoff_factor=1.1) gcs_file = gcs.open(filename, 'w', content_type='text/plain', retry_params=write_retry_params) gcs_file.write(the_feed.encode('UTF-8')) gcs_file.close() return the_feed
def _write_file(self, bucketFile, mtype=None, content="", raw=False, private=False, extrameta=None): log.info("Attempting to write: %s %s %s" % (bucketFile, mtype, raw)) # The retry_params specified in the open call will override the default # retry params for this particular file handle. setAppVar(CLOUDSTAT,None) moremeta = getAppVar(CLOUDEXTRAMETA) setAppVar(CLOUDEXTRAMETA,None) #clear out now potentially stale values if extrameta and moremeta: extrameta.update(moremeta) else: extrameta = moremeta try: write_retry_params = cloudstorage.RetryParams(backoff_factor=1.1) write_options = {} if not private: write_options.update({'x-goog-acl': 'public-read'}) if extrameta: write_options.update(extrameta) if private: write_options="" if not raw: log.info("Encoding to utf8") content = content.encode('utf-8') with cloudstorage.open( bucketFile, 'w', content_type=mtype, options=write_options, retry_params=write_retry_params) as cloudstorage_file: cloudstorage_file.write(content) except Exception as e: log.error("File write error: (%s): %s" % (bucketFile,e)) log.error(traceback.format_exc()) return False setAppVar(CLOUDSTAT,self._stat_file(bucketFile,cache=False)) return True
def get_calfeed_for_key(prefix, the_key): bucket_name = os.environ.get('BUCKET_NAME', app_identity.get_default_gcs_bucket_name()) filename = "/{0}/{1}{2}{3}".format(bucket_name,prefix,"cal-",the_key.urlsafe()) gcs_file = None try: gcs_file = gcs.open(filename) except: gcs_file = None if gcs_file: feed = gcs_file.read() gcs_file.close() logging.info("found file") else: feed = None logging.info("did not find file") return feed
def save_to_gcs(file_obj): serving_url = '' #just assign it adn reassign later time_stamp = int(time.time()) app_id = app_identity.get_application_id() fname = '/%s.appspot.com/post_%s.jpg' % (app_id, time_stamp) logging.error(fname) # Content Types # audio/mpeg # image/jpeg gcs_file = gcs.open(fname, 'w', content_type="image/jpeg") gcs_file.write(file_obj) gcs_file.close() gcs_filename = "/gs%s" % fname serving_url = images.get_serving_url(blobstore.create_gs_key(gcs_filename)) media_obj = save_gcs_to_media(gcs_filename, serving_url) return media_obj
def post(self): datastore_hooks.SetPrivilegedRequest() try: params = json.loads(self.request.body) gcs_file_path = params['gcs_file_path'] try: gcs_file = cloudstorage.open( gcs_file_path, 'r', retry_params=_RETRY_PARAMS) with DecompressFileWrapper(gcs_file) as decompressing_file: histogram_dicts = _LoadHistogramList(decompressing_file) gcs_file.close() ProcessHistogramSet(histogram_dicts) finally: cloudstorage.delete(gcs_file_path, retry_params=_RETRY_PARAMS) except Exception as e: # pylint: disable=broad-except logging.error('Error processing histograms: %r', e.message) self.response.out.write(json.dumps({'error': e.message}))
def post(self): self.response.headers.add_header('Access-Control-Allow-Origin', '*') self.response.headers['Content-Type'] = 'application/json' bucket_name = app_identity.get_default_gcs_bucket_name() uploaded_file = self.request.POST.get('uploaded_file') file_name = getattr(uploaded_file, 'filename', None) file_content = getattr(uploaded_file, 'file', None) real_path = '' if file_name and file_content: content_t = mimetypes.guess_type(file_name)[0] real_path = os.path.join('/', bucket_name, file_name) with cloudstorage.open(real_path, 'w', content_type=content_t, options={'x-goog-acl': 'public-read'}) as f: f.write(file_content.read()) key = self._get_urls_for(file_name) self.response.write(key)
def get(self): days = self.request.get('days', '1') participantID = self.request.get('participantID', '000') today = datetime.today()-timedelta(hours=timezone) userIndex = getUserID(participantID.encode('utf-8')) #self.response.write('userIndex: ' + str(userIndex)) data = {} if (userIndex == 'error'): self.response.set_status(404) self.response.out.write('wrong participantID') return else: bucket_name = os.environ.get( 'BUCKET_NAME', app_identity.get_default_gcs_bucket_name()) bucket = '/' + bucket_name + '/Users' + '/' + userIndex + '/Dailyrecords' for x in range(int(days)): tempday = str((today - timedelta(x)).strftime('%Y%m%d')) tempdayfile = tempday + '/' # self.response.out.write(tempday) if self.FileExists(bucket, tempdayfile): filename = bucket + '/' + tempdayfile + 'weight.json' with cloudstorage.open(filename) as gcs_file_read: contents = gcs_file_read.read() contents_json = json.loads(contents) # self.response.write(type(contents_json["body"]["measuregrps"]).__name__) # self.response.write('\n\n##\n\n') # self.response.write(contents_json["body"]["measuregrps"]) for value in contents_json["body"]["measuregrps"]: #self.response.write(type(value).__name__) #dict #self.response.write(type(value["measures"]).__name__) #list #self.response.write(len(value["measures"]))#1 try: data[str(value["grpid"])]=value["measures"][0]["value"] except KeyError: self.response.write('error') #data[value["grpid"]] = value["measures"]["value"] # for key, value in contents_json["body"]["measuregrps"].iteritems(): # data[value["grpid"]]=value["measures"]["value"] self.response.out.write(data)
def get(self): # Get the default Cloud Storage Bucket name and create a file name for # the object in Cloud Storage. bucket = app_identity.get_default_gcs_bucket_name() # Cloud Storage file names are in the format /bucket/object. filename = '/{}/blobstore_serving_demo'.format(bucket) # Create a file in Google Cloud Storage and write something to it. with cloudstorage.open(filename, 'w') as filehandle: filehandle.write('abcde\n') # In order to read the contents of the file using the Blobstore API, # you must create a blob_key from the Cloud Storage file name. # Blobstore expects the filename to be in the format of: # /gs/bucket/object blobstore_filename = '/gs{}'.format(filename) blob_key = blobstore.create_gs_key(blobstore_filename) # BlobstoreDownloadHandler serves the file from Google Cloud Storage to # your computer using blob_key. self.send_blob(blob_key)
def start_prune_job( job_id): # , email, tree_base, tree_set, sample_size, names with gcs.open(JOB_CONFIG_PATH % job_id) as fh: job_info = yaml.load(fh) # init_job(job_id, email, tree_base, tree_set, sample_size, names) # Get the base tree from config tree_base = job_info['base_tree'].lower() # Get the tree_set from config tree_set = job_info['tree_set'] for k, v in TREE_CODES.iteritems(): if v == tree_set: tree_set = k names = [n.replace(' ', '_') for n in job_info['names']] sample_trees = job_info['sample_trees'] start_pruning(job_id, tree_base, sample_trees, names) # process_bad_names(job_id) finalise_job(job_id, sample_trees, tree_base, tree_set)
def create_test_content(self, bucket_name, object_prefix, num_files): """Create a file in Google Cloud Storage with a small amount of content. Args: bucket_name: the name of the bucket, with no delimiters. object_prefix: a string prefix for each object/file that will be created. A suffix with a file number will automatically be appended. num_files: the number of files to create. Returns: A list with each element containing the data in one of the created files. """ created_content = [] for file_num in range(num_files): content = "Dummy Content %d" % file_num created_content.append(content) test_file = cloudstorage.open( "/%s/%s%03d" % (bucket_name, object_prefix, file_num), mode="w") test_file.write(content) test_file.close() return created_content
def create_file(self, filename): """Create a file. The retry_params specified in the open call will override the default retry params for this particular file handle. Args: filename: filename. """ self.response.write('Creating file %s\n' % filename) write_retry_params = gcs.RetryParams(backoff_factor=1.1) gcs_file = gcs.open(filename, 'w', content_type='text/plain', options={'x-goog-meta-foo': 'foo', 'x-goog-meta-bar': 'bar'}, retry_params=write_retry_params) gcs_file.write('abcde\n') gcs_file.write('f'*1024*4 + '\n') gcs_file.close() self.tmp_filenames_to_clean_up.append(filename)
def post(self): trace_uuid = str(uuid.uuid4()) bucket_name = ('/performance-insights/' + trace_uuid) gcs_file = gcs.open(bucket_name, 'w', content_type='application/octet-stream', options={}, retry_params=default_retry_params) gcs_file.write(self.request.get('trace')) gcs_file.close() trace_object = trace_info.TraceInfo(id=trace_uuid) trace_object.prod = self.request.get('prod') trace_object.remote_addr = os.environ["REMOTE_ADDR"] tags_string = self.request.get('tags') if re.match('^[a-zA-Z0-9,]+$', tags_string): # ignore non alpha-numeric tags trace_object.tags = tags_string.split(',') trace_object.user_agent = self.request.headers.get('User-Agent') trace_object.ver = self.request.get('product_version') trace_object.put() self.response.write(trace_uuid)
def get(self): days = self.request.get('days', '1') participantID = self.request.get('participantID', '000') today = datetime.today()-timedelta(hours=timezone) userIndex = getUserID(participantID.encode('utf-8')) totalstepcount=0 #self.response.out.write('totalstepcount\n') if (userIndex == 'error'): self.response.set_status(404) self.response.out.write('wrong participantID') return else: #self.response.write(userIndex) bucket_name = os.environ.get( 'BUCKET_NAME', app_identity.get_default_gcs_bucket_name()) bucket = '/' + bucket_name + '/Users' + '/' + userIndex + '/Dailyrecords' for x in range (int(days)): tempday = str((today-timedelta(x)).strftime('%Y%m%d'))+'/' #self.response.out.write(tempday) if self.FileExists(bucket,tempday): filename = bucket + '/' + tempday + 'activity.json' with cloudstorage.open(filename) as gcs_file_read: contents = gcs_file_read.read() contents_json = json.loads(contents) #self.response.out.write ('type:' + str(type(contents_json["body"]["series"]).__name__)) if len(contents_json["body"]["series"])!= 0: #self.response.write(len(contents_json["body"]["series"])) for key, value in contents_json["body"]["series"].iteritems(): try: totalstepcount+=int(value["steps"]) except KeyError: self.response.out.write('') gcs_file_read.close() # else: #this is when "days" is larger than the actual number of days in the database # self.response.out.write('') self.response.out.write(totalstepcount)
def migrate_single_blob_inline(blob_info, bucket_name): """Migrates a single, small blob. Args: blob_info: The BlobInfo for the blob to copy. bucket_name: The name of the bucket to copy the blob info. Returns: The resulting filename for the GCS file, rooted by "/[bucket_name]/..." """ options = {} if blob_info.filename: options['content-disposition'] = (build_content_disposition( blob_info.filename.encode('utf8'))) gcs_filename = build_gcs_filename(blob_info, filename=blob_info.filename, bucket_name=bucket_name, include_bucket=True, include_leading_slash=True) blob_reader = blobstore.BlobReader(blob_info, buffer_size=BLOB_BUFFER_SIZE) gcs_file = cloudstorage.open(gcs_filename.encode('utf8'), mode='w', content_type=blob_info.content_type, options=options) try: chunk = blob_reader.read(BLOB_BUFFER_SIZE) while chunk: gcs_file.write(chunk) chunk = blob_reader.read(BLOB_BUFFER_SIZE) finally: gcs_file.close() store_mapping_entity(blob_info, gcs_filename) return gcs_filename
def post(self): if self.session.get('userid') == None: self.redirect('/') message = "" fileitem = self.request.POST["filedata"] if fileitem.filename: fn = os.path.basename(fileitem.filename) filedata = fileitem.file.read() cloudfile = str(uuid.uuid4()) write_retry_params = gcs.RetryParams(backoff_factor=1.1) gcs_file = gcs.open('/' + bucket_name + '/' + cloudfile, 'w', content_type='text/plain', options={ 'x-goog-meta-foo': 'foo', 'x-goog-meta-bar': 'bar' }, retry_params=write_retry_params) gcs_file.write(filedata) gcs_file.close() filelist = FileList(userid=str(self.session['userid']), name=fn, size=len(filedata), loadtime=datetime.datetime.now(), c_position=cloudfile, deleted=False) filelist.put() self.redirect('/dashboard') else: message = 'Dosya Yükleme Hatası' self.response.out.write(message)
def read_file(bucket, filename, chunk_size=CHUNK_SIZE): """Reads a file and yields its content in chunks of a given size. Arguments: bucket: a bucket that contains the file. filename: name of the file to read. chunk_size: maximum size of a chunk to read and yield. Yields: Chunks of a file (as str objects). """ path = '/%s/%s' % (bucket, filename) bytes_read = 0 data = None file_ref = None try: with cloudstorage.open(path, read_buffer_size=chunk_size, retry_params=_make_retry_params()) as file_ref: while True: data = file_ref.read(chunk_size) if not data: break bytes_read += len(data) yield data # Remove reference to a buffer so it can be GC'ed. data = None except Exception as exc: logging.warning('Exception while reading \'%s\', read %d bytes: %s %s', path, bytes_read, exc.__class__.__name__, exc) raise finally: # Remove lingering references to |data| and |file_ref| so they get GC'ed # sooner. Otherwise this function's frame object keeps references to them, # A frame object is around as long as there are references to this # generator instance somewhere. data = None file_ref = None
def create(cls, mapreduce_state, shard_state): """Create new writer for a shard. Args: mapreduce_state: an instance of model.MapreduceState describing current job. State can NOT be modified. shard_state: an instance of model.ShardState. Returns: an output writer for the requested shard. """ # Get the current job state job_spec = mapreduce_state.mapreduce_spec writer_spec = _get_params(job_spec.mapper, allow_old=False) # Determine parameters key = cls._generate_filename(writer_spec, job_spec.name, job_spec.mapreduce_id, shard_state.shard_number, shard_state.retries) # GoogleCloudStorage format for filenames, Initial slash is required filename = "/%s/%s" % (writer_spec[cls.BUCKET_NAME_PARAM], key) content_type = writer_spec.get(cls.CONTENT_TYPE_PARAM, None) options = {} if cls.ACL_PARAM in writer_spec: options["x-goog-acl"] = writer_spec.get(cls.ACL_PARAM) account_id = writer_spec.get(cls._ACCOUNT_ID_PARAM, None) writer = cloudstorage.open(filename, mode="w", content_type=content_type, options=options, _account_id=account_id) return cls(writer, filename, writer_spec=writer_spec)
def _get_file(self, bucketFile, reqtype=None, cache=True): log.info("_get_file(%s,%s)" % (bucketFile,cache)) setAppVar(CLOUDSTAT,None) stat = None content = None cached = False if cache: item = self.readCache(bucketFile,reqtype) if item: content = item.content stat = item.stat if content: cached = True log.info("Got from readCache") if not stat: stat = self._stat_file(bucketFile,cache=False) if stat: log.info('Opening file {}'.format(bucketFile)) try: with cloudstorage.open(bucketFile) as cloudstorage_file: content = cloudstorage_file.read() cloudstorage_file.close() except cloudstorage.NotFoundError: log.info("File not found: %s" % bucketFile) except Exception as e: log.info("File read error (%s): %s" % (bucketFile,e)) if not cached and content: log.info("Adding to cache: %s" % bucketFile) val = bucketCacheItem(stat=stat,content=content) self.writeCache(bucketFile,val, reqtype) setAppVar(CLOUDSTAT,stat) return stat, content
def post(self): datastore_hooks.SetPrivilegedRequest() try: params = json.loads(self.request.body) gcs_file_path = params['gcs_file_path'] try: gcs_file = cloudstorage.open( gcs_file_path, 'r', retry_params=_RETRY_PARAMS) contents = gcs_file.read() data_str = zlib.decompress(contents) gcs_file.close() finally: cloudstorage.delete(gcs_file_path, retry_params=_RETRY_PARAMS) with timing.WallTimeLogger('json.loads'): histogram_dicts = json.loads(data_str) ProcessHistogramSet(histogram_dicts) except Exception as e: # pylint: disable=broad-except logging.error('Error processing histograms: %r', e.message) self.response.out.write(json.dumps({'error': e.message}))
def get(self): app_id = self.request.get("app_id", None) azzert(app_id is not None, "app_id is not found") logging.debug("GetOSALaucherAppHandler app_id: %s", app_id) app = OSALauncherApp.get_by_app_id(app_id) if app: filename = "%s-%s.apk" % (app.app_id, app.version_code) try: gae_filename = '%s/oca/launcher/apps/%s.apk' % ( ROGERTHAT_ATTACHMENTS_BUCKET, app_id) self.response.headers[ 'Content-Type'] = "application/vnd.android.package-archive" self.response.headers['Content-Disposition'] = str( 'attachment; filename=%s' % filename) with cloudstorage.open(gae_filename, 'r') as gcs_file: self.response.write(gcs_file.read()) except cloudstorage.errors.NotFoundError: logging.warn("GetOSALaucherAppHandler NOT found in gcs") self.error(500) else: self.error(500)
def post(self): file = open('header.html') self.response.write(file.read()) bucket_name = app_identity.get_default_gcs_bucket_name() self.response.write("<h4>"+bucket_name+"</h4>") name = self.request.POST.get('tfName') branch = self.request.POST.get('ddBranch') self.response.write(name) self.response.write(branch) data = self.request.POST.getall('file1') self.response.write(data) self.response.write("<hr/>") for file in data: f1 = gcs.open("/"+bucket_name+"/"+file.filename, "w", content_type=file.type) f1.write(file.file.read()) f1.close() self.response.write("<h4>https://storage.googleapis.com/"+bucket_name+"/"+file.filename+"</h4>") self.response.write("Files uploaded successfully")
def serve(self, d): import cloudstorage as gcs rkey = self.request.get('rkey') r = Report.GetAccessible(rkey, self.user, urlencoded_key=True) if r: if r.is_done() and r.gcs_files: gcsfn = r.gcs_files[0] if tools.on_dev_server(): try: gcs_file = gcs.open(gcsfn, 'r') except gcs.NotFoundError, e: self.response.out.write("File not found") else: self.response.headers['Content-Type'] = Report.content_type(r.extension) self.response.headers['Content-Disposition'] = str('attachment; filename="%s"' % r.filename()) self.response.write(gcs_file.read()) gcs_file.close() else: signed_url = tools.sign_gcs_url(gcsfn, expires_after_seconds=5) response = self.redirect(signed_url) logging.info(response) else: self.set_response(success=False, status=404, message="Report not ready") # Not found
def get_last_end_time(project_id, bucket_name): """ Get the end_time as a string value from a JSON object in GCS. This file is used to remember the last end_time in case one isn't provided """ last_end_time_str = "" file_name = '{}.{}'.format(project_id, config.LAST_END_TIME_FILENAME) logging.debug("get_last_end_time - file_name: {}".format(file_name)) try: gcs_file = gcs.open('/{}/{}'.format( bucket_name, file_name)) contents = gcs_file.read() logging.debug("GCS FILE CONTENTS: {}".format(contents)) json_contents = json.loads(contents) last_end_time_str = json_contents["end_time"] gcs_file.close() except NotFoundError as nfe: logging.error("Missing file when reading {} from GCS: {}".format(file_name, nfe)) last_end_time_str = None except Exception as e: logging.error("Received error when reading {} from GCS: {}".format(file_name,e)) last_end_time_str = None return last_end_time_str
def playsong(request, song_id, song_seed): '''Grabs current song to get json. Gets seed from last page seed value and magic''' song = get_object_or_404(UMusic, user=request.user, id=song_id) song_json = json.loads(json.loads(song.song_json)) inFiles = getwavs(song_json, song_seed) # get paths from algorythm if settings.INPRODUCTION: songData = "" for inFile in inFiles: file = gcs.open(inFile) songData += file.read() file.close() else: songData = "" for inFile in inFiles: file = open(inFile, "rb") songData += file.read() file.close() return HttpResponse(songData, content_type='audio/mpeg')
def post(self, user): self.response.headers['Content-Type'] = 'text/plain' self.response.out.write('All OK!!') bucket = '/modelr_live_bucket/' filename = bucket + str(user.user_id) + '/' + str(time.time()) encoded_image = self.request.get('image').split(',')[1] pic = base64.b64decode(encoded_image) gcsfile = gcs.open(filename, 'w') gcsfile.write(pic) gcsfile.close() bs_file = '/gs' + filename blob_key = blobstore.create_gs_key(bs_file) ImageModel(parent=user, user=user.user_id, image=blob_key).put()
def _open_file(cls, writer_spec, filename_suffix, use_tmp_bucket=False): """Opens a new gcs file for writing.""" if use_tmp_bucket: bucket = cls._get_tmp_gcs_bucket(writer_spec) account_id = cls._get_tmp_account_id(writer_spec) else: bucket = cls._get_gcs_bucket(writer_spec) account_id = cls._get_account_id(writer_spec) # GoogleCloudStorage format for filenames, Initial slash is required filename = "/%s/%s" % (bucket, filename_suffix) content_type = writer_spec.get(cls.CONTENT_TYPE_PARAM, None) options = {} if cls.ACL_PARAM in writer_spec: options["x-goog-acl"] = writer_spec.get(cls.ACL_PARAM) return cloudstorage.open(filename, mode="w", content_type=content_type, options=options, _account_id=account_id)
def AuthenticatedGet(self, org, event): filename = all_event_timeless_filename(event) bucket_path = BUCKET_NAME + '/' + filename try: file_stat = cloudstorage.stat(bucket_path) except cloudstorage.NotFoundError: self.abort(404) # rewrite filename to include timestamp custom_timestamp = timestamp( datetime.datetime.utcfromtimestamp(file_stat.st_ctime)) filename_to_serve = file_stat.filename.replace( '.csv', '-%s.csv' % custom_timestamp) # serve the file as an attachment, forcing download gcs_fd = cloudstorage.open(bucket_path) if file_stat.content_type: self.response.headers['Content-Type'] = file_stat.content_type self.response.headers['Content-Disposition'] = (str( 'attachment; filename="%s"' % filename_to_serve)) self.response.write(gcs_fd.read())