Python get_file_to_mem примеры использования

Пример #1

Файл: analyze_main.py Проект: dongsam/works

def analyze_oneday_log_s3(daystr, outf=''):
	print "Output: %s" %(outf)
	analyzer = queryClickLog.LogAnalyzer(daystr)

	for qfile in make_q_names(daystr):
		print qfile
		org_size, mem = handolS3Util.get_file_to_mem( handolS3Util.conn, BUCKET, qfile)
		print "ORG [%d] LOAD [%d]" % (org_size, len(mem))
		if org_size==0 or org_size != len(mem):
			print "!!! NOT fully loaded"
			continue
		analyzer.query_log_mem(mem)
		sys.stdout.flush()

	for cfile in make_c_names(daystr):
		print cfile
		org_size, mem = handolS3Util.get_file_to_mem( handolS3Util.conn, BUCKET, cfile)
		print "ORG [%d] LOAD [%d]" % (org_size, len(mem))
		if org_size==0 or org_size != len(mem):
			print "!!! NOT fully loaded"
			continue
		analyzer.click_log_mem(mem)
		sys.stdout.flush()

	analyzer.get_rate()
	analyzer.print_info(outf)

Пример #2

Файл: analyze_main.py Проект: eox03y/works

def analyze_oneday_log_s3(daystr, outf=''):
    print "Output: %s" % (outf)
    analyzer = queryClickLog.LogAnalyzer(daystr)

    for qfile in make_q_names(daystr):
        print qfile
        org_size, mem = handolS3Util.get_file_to_mem(handolS3Util.conn, BUCKET,
                                                     qfile)
        print "ORG [%d] LOAD [%d]" % (org_size, len(mem))
        if org_size == 0 or org_size != len(mem):
            print "!!! NOT fully loaded"
            continue
        analyzer.query_log_mem(mem)
        sys.stdout.flush()

    for cfile in make_c_names(daystr):
        print cfile
        org_size, mem = handolS3Util.get_file_to_mem(handolS3Util.conn, BUCKET,
                                                     cfile)
        print "ORG [%d] LOAD [%d]" % (org_size, len(mem))
        if org_size == 0 or org_size != len(mem):
            print "!!! NOT fully loaded"
            continue
        analyzer.click_log_mem(mem)
        sys.stdout.flush()

    analyzer.get_rate()
    analyzer.print_info(outf)

Пример #3

Файл: qcmerger.py Проект: dongsam/works

def load_from_s3(bucket, s3file):
	org_size, mem = handolS3Util.get_file_to_mem( handolS3Util.conn, bucket, s3file)
	print "Open S3: %s/%s" % (bucket, s3file)
	print "ORG [%d] LOAD [%d]" % (org_size, len(mem))
	if org_size==0 or org_size != len(mem):
		print "Failed:", s3file
		return None
	else:
		return mem

Пример #4

def load_from_s3(bucket, s3file):
    org_size, mem = handolS3Util.get_file_to_mem(handolS3Util.conn, bucket,
                                                 s3file)
    print "Open S3: %s/%s" % (bucket, s3file)
    print "ORG [%d] LOAD [%d]" % (org_size, len(mem))
    if org_size == 0 or org_size != len(mem):
        print "Failed:", s3file
        return None
    else:
        return mem

Пример #5

Файл: queryKeywordRank.py Проект: eox03y/works

	def querylog_from_s3(self, bucket, s3file):
		org_size, mem = handolS3Util.get_file_to_mem( handolS3Util.conn, bucket, s3file)
		print "Open S3: %s/%s" % (bucket, s3file)
		print "ORG [%d] LOAD [%d]" % (org_size, len(mem))
		if org_size==0 or org_size != len(mem):
			print "Failed:", s3file
			return

		res = self.querylog_from_memory(mem)
		self.fileList[s3file] = res