コード例 #1
0
ファイル: hostSocket.py プロジェクト: hacder/src
def FproSocket(host, port, cmd, type=None):
	'''
	@host: 主机IP
	@port: 主机端口
	@cmd:  指令
	'''

	# 开始通信
	try:
		sock = socket.socket()
		sock.settimeout(SOCKET['PROTIMEOUT'])
		sock.connect((host, port))
		if type:
			sock.send(encode(cmd + type))
		else:
			if host == '127.0.0.1':
				sock.send(cmd)
			else:
				sock.send(encode(cmd))
		
		sock.close()

	except Exception as e:
		save_log('ERROR',str(e))
		return False
	
	return 1
コード例 #2
0
def main():
	user_input = int(raw_input("Welcome \
to steganography.\nEnter 1 for encoding \
an image or 2 for decoding an image.\n"))
	if user_input == 1:
		print "Message length must have a length less than 255."
		encode_image = raw_input("Please enter the \
image you would like to encode your message \
into.\n")
		hidden_text = raw_input("Please enter the \
text file that contains the message you \
would like to send.\n")
		dest_image = raw_input("Please enter the \
filename you would like the encoded image \
to be called.\n")
		encode(encode_image, hidden_text, dest_image)
		user_email = int(raw_input("Would you like to send this in an email?\n\
If so, enter 1. If not, enter any other character.\n"))
		if user_email == 1:
			sendEmail(dest_image);
	elif user_input == 2:
		decode_image = raw_input("Please enter the \
image you would like to decode.\n")
		decode(decode_image)
	else:
		print "You have entered an invalid option."
コード例 #3
0
def metropolis_hastings(mc, doc):
    #initialize randomized T
    t = gen_t()
    #consider starting fresh after k iterations
    for i in range(20000):
        #t = t if (i%2000) != 0 else gen_t()
        #if i % 10000:
        #new_seed = gen_t()
        #if abs(logprob(mc, encode.encode(doc, new_seed[0], new_seed[1]))) < abs(logprob(mc, encode.encode(doc, t[0], t[1]))):
        #t = new_seed

        if random.randrange(100) < 99:
            tp = mod_sub(copy.deepcopy(t))
        else:
            tp = mod_transl(copy.deepcopy(t))

        logprob_decoded_t = logprob(mc, encode.encode(doc, t[0], t[1]))
        logprob_decoded_tp = logprob(mc, encode.encode(doc, tp[0], tp[1]))

        # logprob is closer to zero if probability is larger
        # check to make sure comparison is right
        if abs(logprob_decoded_tp) < abs(logprob_decoded_t):
            t = copy.deepcopy(tp)
        else:
            ratio = math.exp(logprob_decoded_tp - logprob_decoded_t)
            if random.random() > ratio:
                t = copy.deepcopy(t)
            else:
                t = copy.deepcopy(tp)

    return t
コード例 #4
0
def break_code(string, corpus, replace_table, rearrange_table, t):
    corpus_trans_prob = get_tran_prob(corpus)
    decode_string = encode.encode(string, replace_table, rearrange_table)
    prob_decode_d = caluclate_prob(corpus_trans_prob, decode_string)
    count = 0
    re_pl = {}
    re_ar = {}
    t1 = time.time()
    while time.time() - t1 < t:
        count += 1
        replace_table1, rearrange_table1 = get_updated_tables(
            replace_table, rearrange_table, corpus_trans_prob)
        decode_string1 = encode.encode(string, replace_table1,
                                       rearrange_table1)
        prob_decode_d1 = caluclate_prob(corpus_trans_prob, decode_string1)

        if prob_decode_d > prob_decode_d1:
            prob = np.random.binomial(
                1, np.exp((prob_decode_d1 - prob_decode_d) / 100))
            if (prob == 1):
                replace_table, rearrange_table = copy.deepcopy(
                    replace_table1), copy.deepcopy(rearrange_table1)
                prob_decode_d = prob_decode_d1

        else:
            re_pl[prob_decode_d1] = replace_table1
            re_ar[prob_decode_d1] = rearrange_table1
            replace_table, rearrange_table = copy.deepcopy(
                replace_table1), copy.deepcopy(rearrange_table1)
            prob_decode_d = prob_decode_d1

    print(count)
    return decode_string, re_pl, re_ar
コード例 #5
0
ファイル: tmp.py プロジェクト: Eragon666/ModerneDatabases
    def _get_data(self):
        """
        Call the _commit() methods of the children nodes. And return the encoded
        data of the node, which contains its type, and the offsets of the
        children nodes.
        """

        data = {}
        if self.bucket != None:
            for (key, value) in self.bucket.items():
                data[key] = value._commit()

        if self.rest != None:
            rest_data = self.rest._commit()
            # print("Node committed: " + str(self)+ " bucketsize: " +
            #     str(len(self.bucket)))
            # print("Node data: " +
            #     str({"type":"Node", "rest":rest_data, "entries":data}))
            return add_integrity(encode({"type":"Node", "rest":rest_data,
                "entries":data}))

        # print("Node committed: " + str(self)+ " bucketsize: " +
        #     str(len(self.bucket)))
        # print("Node data: "+ str({"type":"Node", "entries":data}))
        return add_integrity(encode({"type":"Node", "entries":data}))
コード例 #6
0
def FproSocket(host, port, cmd, type=None):
    '''
	@host: 主机IP
	@port: 主机端口
	@cmd:  指令
	'''

    # 开始通信
    try:
        sock = socket.socket()
        sock.settimeout(SOCKET['PROTIMEOUT'])
        sock.connect((host, port))
        if type:
            sock.send(encode(cmd + type))
        else:
            if host == '127.0.0.1':
                sock.send(cmd)
            else:
                sock.send(encode(cmd))

        sock.close()

    except Exception as e:
        save_log('ERROR', str(e))
        return False

    return 1
コード例 #7
0
def break_the_code(string, corpus):
    #Generate initial replace and rearrangement tables
    letters = list(range(ord('a'), ord('z') + 1))
    random.shuffle(letters)
    replace_table = dict(
        zip(map(chr, range(ord('a'),
                           ord('z') + 1)), map(chr, letters)))
    rearrange_table = list(range(0, 4))
    random.shuffle(rearrange_table)

    modified = 0
    cnt = 0
    end_encryption = []

    corpus_W0, corpus_W1 = generate_probs(corpus)
    best_score = score_my_file(
        corpus_W0, corpus_W1,
        encode.encode(string, replace_table, rearrange_table))
    # i = 0
    while True:
        # i+=1
        if np.random.uniform() > 0.5:
            modified = 1
            rearrange_table_backup = deepcopy(rearrange_table)
            rearrange_table = deepcopy(generate_rearrangement(rearrange_table))
        else:
            modified = 0
            replace_table_backup = deepcopy(replace_table)
            replace_table = deepcopy(generate_replacement(replace_table))

        score = score_my_file(
            corpus_W0, corpus_W1,
            encode.encode(string, replace_table, rearrange_table))

        if score > best_score:
            cnt += 1
            best_score = score  #Replace the best_score with the new score
        else:  #If not, replace with probability P(D')/P(D)
            if np.random.binomial(1, np.exp(score - best_score)) == 0:
                if modified == 1:
                    rearrange_table = deepcopy(
                        rearrange_table_backup)  #Revert and not replace
                else:
                    replace_table = deepcopy(
                        replace_table_backup)  #Revert and not replace
            else:
                cnt += 1
                best_score = score  #Replace
        # print('Iter:',i,'Count:',cnt)

        end_encryption.append(cnt)
        #If there have been at least 20k iterations and the last 1000 entries have not changed then abort
        if len(end_encryption) > 20000:
            all_same = end_encryption[len(end_encryption) -
                                      1000:len(end_encryption)]
            if all(x == all_same[0] for x in all_same):
                break
    final_decryption = encode.encode(string, replace_table, rearrange_table)
    return final_decryption
コード例 #8
0
def agreement(P, config={}):
    '''
    P: 导频数
    config: 密钥生成配置项,包括:
        sampling_period:采样周期
        sampling_time:采样时间
        corr_ab: Alice和Bob的信道测量值的相关系数
        corr_ae: Alice和Eve的信道测量值的相关系数
        block_size:双阈值量化的子块采样点数
        coef: 双阈值量化的量化系数
        qtype: 均匀量化的编码方式。gray/natural
        order: 均匀量化的量化阶数
        mtype: 合并类型。RSSI/Phase/cross/and/or/xor
        iteration: 信息协调迭代次数
        m: 汉明码监督位m
    '''
    ''' 采样参数 '''
    sampling_period = config.get('sampling_period', 1)
    sampling_time = config.get('sampling_time', 3)
    corr_ab = config.get('corr_ab', 0.9)
    corr_ae = config.get('corr_ae', 0.4)
    ''' 量化参数 '''
    block_size = config.get('block_size', 25)
    coef = config.get('coef', 0.8)
    qtype = config.get('qtype', 'gray')
    order = config.get('order', 1)
    mtype = config.get('mtype', 'cross')
    ''' 信息协调参数 '''
    iteration = config.get('iteration', 2)
    m = config.get('m', 3)
    ''' 采样 '''
    rssi_A, rssi_B, rssi_E = sampling_RSSI(sampling_period, sampling_time,
                                           corr_ab, corr_ae)
    phase_A, phase_B, phase_E = sampling_phase(sampling_period, sampling_time,
                                               corr_ab, corr_ae)
    ''' RSSI量化 '''
    bits_A_rssi, drop_list_A = quantize_ASBG_1bit(rssi_A, block_size, coef)
    bits_B_rssi, drop_list_B = quantize_ASBG_1bit(rssi_B, block_size, coef)
    bits_E_rssi, drop_list_E = quantize_ASBG_1bit(rssi_E, block_size, coef)
    bits_A_rssi = remain(bits_A_rssi, drop_list_B)
    bits_B_rssi = remain(bits_B_rssi, drop_list_A)
    bits_E_rssi = remain(bits_E_rssi, drop_list_A)
    ''' Phase量化 '''
    bits_A_phase = quantize_phase(phase_A, qtype, order)
    bits_B_phase = quantize_phase(phase_B, qtype, order)
    bits_E_phase = quantize_phase(phase_E, qtype, order)
    ''' 合并 '''
    bits_A = merge(bits_A_rssi, bits_A_phase, mtype)
    bits_B = merge(bits_B_rssi, bits_B_phase, mtype)
    bits_E = merge(bits_E_rssi, bits_E_phase, mtype)
    ''' 信息协调 '''
    bits_A, bits_B = winnow(bits_A, bits_B, iteration, m)
    ''' 生成导频 '''
    pos_A = encode(bits_A, P)
    pos_B = encode(bits_B, P)
    pos_E = encode(bits_E, P)

    return pos_A, pos_B, pos_E
コード例 #9
0
    def commit(self):
        with open(self.filename, 'ba') as db:
            self.root._commit(db)
            pos = db.tell()

            db.write(encode(self))
            footer = {'type': 'footer', 'tree': pos}

            db.write(encode(footer))
コード例 #10
0
def main():
    # input flags for this program
    parser = OptionParser()
    parser.add_option("-m",
                      "--message",
                      action="store_true",
                      dest="encode_message",
                      default=True,
                      help="encode a message from the command line")
    parser.add_option("-e",
                      "--encode",
                      dest="encode_file",
                      help="text file to with message to encode and send")
    parser.add_option("-o",
                      "--output",
                      dest="output_file",
                      help="output file with encoded audio")
    parser.add_option("-d",
                      "--decode",
                      dest="decode_file",
                      help="wav file to decode")
    parser.add_option("-p",
                      "--play",
                      action="store_true",
                      dest="play_audio",
                      default=False,
                      help="play audio after encoding text message")
    parser.add_option("-l",
                      "--listen",
                      action="store_true",
                      dest="listen",
                      default=False,
                      help="listen for audio signal and decode")
    options, args = parser.parse_args()

    if options.encode_file:
        output_file = options.output_file if options.output_file else 'output.wav'
        encode(options.encode_file, output_file)
        if options.play_audio:
            play_audio(output_file)
    elif options.listen:
        return listen()
    elif options.decode_file:
        decode(options.decode_file)
    elif options.encode_message:
        message = raw_input("Type your message: ")
        output_file = "output.wav"
        text_file = open("temp.txt", "w")
        text_file.write(message)
        text_file.close()
        encode("temp.txt", output_file)
        print "* * done! Your audio file is available at 'output.wav'."
        if options.play_audio:
            play_audio(output_file)
    else:
        parser.print_help()
    return
コード例 #11
0
def agreement(P,mtype='cross',iteration=2,corr_ab=0.9,corr_ae=0.4):
    '''
    P: 导频数
    mtype: 合并类型。RSSI/Phase/cross/and/or/xor
    iteration: winnow迭代次数
    corr_ab: Alice和Bob的信道测量值的相关系数
    corr_ae: Alice和Eve的信道测量值的相关系数
    '''
    
    ''' 采样参数 '''
    sampling_period = 1
    sampling_time = 3
    
    ''' 量化参数 '''
    block_size = 25
    coef = 0.8
    qtype = 'gray'
    order = 1
    
    ''' 采样 ''' 
    rssi_A,rssi_B,rssi_E = sampling('RSSI',sampling_period,sampling_time,corr_ab,corr_ae)  
    phase_A,phase_B,phase_E = mod(sampling('Phase',sampling_period,sampling_time,corr_ab,corr_ae),2*pi)
    #print 'corrcoef of rssi  between AB and AE:',corrcoef(rssi_A, rssi_B, rowvar=0)[0,1],corrcoef(rssi_A, rssi_E, rowvar=0)[0,1]
    #print 'corrcoef of phase between AB and AE:',corrcoef(phase_A,phase_B,rowvar=0)[0,1],corrcoef(phase_A,phase_E,rowvar=0)[0,1]
        
    ''' RSSI量化 '''
    bits_A_rssi,drop_list_A = quantization_thre(rssi_A,block_size,coef)
    bits_B_rssi,drop_list_B = quantization_thre(rssi_B,block_size,coef)
    bits_E_rssi,drop_list_E = quantization_thre(rssi_E,block_size,coef)
    bits_A_rssi = remain(bits_A_rssi,drop_list_A,drop_list_B)
    bits_B_rssi = remain(bits_B_rssi,drop_list_A,drop_list_B)
    bits_E_rssi = remain(bits_E_rssi,drop_list_A,drop_list_E)
    #print 'BMR of RSSI before winnow between AB',BMR(bits_A_rssi,bits_B_rssi)
    
    ''' Phase量化 '''
    bits_A_phase = quantization_even('Phase',phase_A,size(phase_A),qtype,order)
    bits_B_phase = quantization_even('Phase',phase_B,size(phase_B),qtype,order)
    bits_E_phase = quantization_even('Phase',phase_E,size(phase_E),qtype,order)
    #print 'BMR of phase before winnow between AB',BMR(bits_A_phase,bits_B_phase)
    
    ''' 合并 '''
    bits_A = merge(bits_A_rssi,bits_A_phase,mtype)
    bits_B = merge(bits_B_rssi,bits_B_phase,mtype)
    bits_E = merge(bits_E_rssi,bits_E_phase,mtype)
    #print 'BMR of merge before winnow between AB',BMR(bits_A,bits_B)
    
    ''' winnow信息协调 '''
    bits_A, bits_B = winnow(bits_A,bits_B,iteration)
    #print 'BMR of merge after winnow between AB',BMR(bits_A,bits_B)
    
    ''' 生成导频 '''
    pos_A = encode(bits_A,P)
    pos_B = encode(bits_B,P)
    pos_E = encode(bits_E,P)

    return pos_A,pos_B,pos_E
コード例 #12
0
ファイル: main.py プロジェクト: meecoder/dummyphone-app
def go():
    test = input("Press 1 for encoding and 2 for decoding, then press Enter. ")

    if (test == "1"):
        encode()
    elif (test == "2"):
        decode()
    else:
        print("Please type 1 or 2. Try again.")
        go()
コード例 #13
0
ファイル: main.py プロジェクト: meecoder/dummyphone-app
def go():
	test = input("Press 1 for encoding and 2 for decoding, then press Enter. ")
	
	if(test == "1"):
		encode()
	elif(test == "2"):
		decode()
	else:
		print("Please type 1 or 2. Try again.")
		go()
コード例 #14
0
ファイル: hostSocket.py プロジェクト: hacder/src
def proSocket(host, port, cmd, type=None):
	'''
	@host: 主机IP
	@port: 主机端口
	@cmd:  指令
	'''

	data = ''
	d = ''
	try:
		sock = socket.socket()
		sock.settimeout(SOCKET['PROTIMEOUT'])
		sock.connect((host, port))
		print encode(cmd)	
		if type:
			sock.send(encode(cmd + type))
		else:
			sock.send(encode(cmd))

		while True:
			d = sock.recv(SOCKET['BUFSIZE'])
			data = data + d
			time.sleep(SOCKET['DELAY'])
			if not d:
				break
		try:
			data = decode(data)
		except  Exception as e:
			
			log_error('decode error:'+str(e) + ' Try increasing the delay.')
			sock.send(encode(cmd))
			
			data=''
			d=''
			
			while True:
				d = sock.recv(SOCKET['BUFSIZE'])
				data = data + d
				time.sleep(SOCKET['DELAY']+1)
				if not d:
					break
			try:
				data = decode(data)
			except  Exception as e:
				
				log_error('recv decode error:'+str(e))
				return False
	except Exception as e:
		
		log_error('ip: '+host+' , port: '+str(port)+' , proSocket other error: '+str(e))
		sock.close()
		return False
	
	return data
コード例 #15
0
ファイル: main.py プロジェクト: cogne/AWGN_channel
def main():
    """ Execute a full transmission of a text file through an Additive Gaussian Noise channel. """

    symbols = read_message(MESSAGE_FILE)
    encode(symbols)

    os.system('python3 ../client/client.py --input_file=' + INPUT_FILE +
              ' --output_file=' + OUTPUT_FILE +
              ' --srv_hostname=iscsrv72.epfl.ch --srv_port=80')

    recovered_symbols = decode()
    write_guess(recovered_symbols, GUESS_FILE)
コード例 #16
0
def embedB(imgB, dataList, Aindex):
    matrix_input = np.asarray(imgB)
    matrix_input = matrix_input.astype(int)
    embedRound = 1
    matrix_output, nextInfoList = encode.encode(matrix_input, dataList, Aindex,
                                                embedRound)
    while len(nextInfoList) != 0:
        embedRound += 1
        print('embedRound: ', embedRound)
        matrix_output, nextInfoList = encode.encode(matrix_output,
                                                    nextInfoList, Aindex,
                                                    embedRound)
    #matrix_output = encode(imgB, dataList)
    return matrix_output
コード例 #17
0
def break_code(string, corpus):
    transition = transition_probability_distribution(corpus)
    initial = initial_probability_distribuition(corpus)
    no_of_iterations = 20000  # how many times sampling will be done to get better decrypted document

    old_replace_table, old_rearrange_table = modify_encryption(
    )  #Stores initial encryption tables
    T_guess = encode.encode(string, old_replace_table,
                            old_rearrange_table)  #encoding
    T_prob = log_probability_of_document(
        T_guess.split(" "), transition,
        initial)  # returns prob for guess with given encryption

    new_replace_table = copy.deepcopy(
        old_replace_table)  #depcopy used to copy table in new table
    new_rearrange_table = copy.deepcopy(old_rearrange_table)
    while (no_of_iterations != 0):
        old_rearrange_table = copy.deepcopy(new_rearrange_table)
        old_replace_table = copy.deepcopy(new_replace_table)
        flag = random.randint(
            0, 1
        )  #generate random number among 0 and 1 to modify one of the two encryption tables
        if (flag == 0):
            new_replace_table = modify_replace(old_replace_table)
        else:
            new_rearrange_table = modify_rearrangement(old_rearrange_table)

        T_hat = encode.encode(
            string, new_replace_table,
            new_rearrange_table)  #encoding with modified encryption tables
        T_hat_prob = log_probability_of_document(
            T_hat.split(" "), transition,
            initial)  # returns prob for guess with given encryption

        if (T_hat_prob > T_prob):  # new guess is better than old
            T_prob = T_hat_prob
        else:  # new guess is not better than old
            if (np.random.binomial(1, np.exp(T_hat_prob - T_prob)) == 0):
                if flag == 1:  #changing one of the two encryption tables
                    new_rearrange_table = copy.deepcopy(old_rearrange_table)
                else:
                    new_replace_table = copy.deepcopy(old_replace_table)
            else:
                T_prob = T_hat_prob
        no_of_iterations -= 1


#    print(math.exp(T_hat_prob),math.exp(T_prob))
    return encode.encode(string, new_replace_table, new_rearrange_table)
コード例 #18
0
ファイル: hostSocket.py プロジェクト: hacder/src
def FproSocket_modKey(host ,port ,cmd ,type=None):
	try:
		sock = socket.socket()
		#sock.settimeout(SOCKET['PROTIMEOUT'])
		sock.connect((host, port))
                
		if type:
			sock.send(encode(cmd + type ,COMMANDS['_MOD_KEY']))
		else:
			sock.send(encode(cmd,COMMANDS['_MOD_KEY']))
		sock.close()
	except Exception as e:
		log_error('FproSocket error:'+str(e))
		return 0
	return 1
コード例 #19
0
ファイル: __main__.py プロジェクト: Emzi0767/Simplybius
def main(args: List[str]):
    if args[0] not in ["encode", "decode", "dencode", "ddecode"]:
        raise ValueError(
            "Operation needs to be encode, decode, dencode, or ddecode.")

    if args[0] == "encode":
        print(" ".join(str(x) for x in encode(" ".join(args[1:]))))

    elif args[0] == "decode":
        print(decode([int(x) for x in args[1:]]))

    elif args[0] == "dencode":
        print(discordify(encode(" ".join(args[1:]))))

    elif args[0] == "ddecode":
        print(decode(undiscordify(" ".join(args[1:]))))
コード例 #20
0
 def test_isEncoderFileCreated(self):
     path = str(pathlib.Path(__file__).resolve().parents[2])
     path = path + "/src/Facial recognition/dataset"
     print(path)
     endcode_obj = encode()
     is_file_created = endcode_obj.run(path)
     self.assertTrue(is_file_created)
コード例 #21
0
ファイル: decode.py プロジェクト: cumulus13/decode_image
 def __init__(self):
     '''
     @summary: initialize
     @result: None
     '''
     super(decode, self)
     self.encode = encode.encode()
コード例 #22
0
def main():
    # Message to encode
    message = str(input("Enter message to encode: "))

    # Create path
    directory = "Pics"
    file_name = "test.png"
    path = os.path.join(directory, file_name)

    # Load image
    image = cv2.imread(path, 1)

    # Show before
    cv2.imshow("before", image)

    # Encode message inside the image
    encoded_image = encode(image, message)

    # Save encoded image
    directory = "Pics"
    file_name = message.split(" ")[0] + ".png"
    path = os.path.join(directory, file_name)
    cv2.imwrite(path, image)

    # Show result
    cv2.imshow("after", encoded_image)

    # Decode image and receive message
    recovered_message = decode(encoded_image)

    print("Message should be the same as input: ", recovered_message)

    cv2.waitKey(0)
コード例 #23
0
def send_messasge(src_host, host, port, covert_msg, plaintext_msg):
	plaintext_msg += udp.EOF
	encoded_input = encode.encode(covert_msg)
	#debug print "encoded input: " + str(encoded_input)
	packet_sender = udp.UDP(host, port)
	msg_length = len(plaintext_msg)
	msg_segment_size = msg_length / len(encoded_input) + 1
	
	min = 0
	max = int(msg_length / msg_segment_size) + 1
	for x in range(min, max):
		# determine what part of the fake message to send in this packet
		start = int(x * msg_segment_size)
		end = int(start + msg_segment_size)
		if end > msg_length:
			end = msg_length

		# determine what should be sent for the covert message
		covert_msg_segment = 0
		if x < len(encoded_input):
			covert_msg_segment = encoded_input[x]
		else:
			covert_msg_segment = random.randint(SOURCE_PORT_MIN_INT, SOURCE_PORT_MAX_INT)

		packet_sender.send(src_host, covert_msg_segment, plaintext_msg[start:end])
コード例 #24
0
ファイル: main.py プロジェクト: LysandreJik/AutoEncoderCrypto
def start_training():
    alice_key, bob_key = dh_exchange()
    print('Key has been chosen !', alice_key, bob_key)
    np.random.seed(alice_key)

    from auto_encoder import train
    from encode import encode, decode

    print('Starting training ...')
    train(Callback2(), set_auto_encoder)

    alice_sentence = "bonjour bob 123456"

    encrypted_sentence = encode(alice_sentence)

    decrypted_sentence = decode(encrypted_sentence)

    print('Original sentence was', alice_sentence)
    print('Encrypted sentence was',
          np.array(encrypted_sentence).reshape((18, 10)))
    img = Image.fromarray(
        np.array(encrypted_sentence).reshape((18, 10)) * 255 /
        np.max(np.max(np.array(encrypted_sentence).reshape((18, 10)))))
    img.show()
    print('Decrypted sentence is', decrypted_sentence)
コード例 #25
0
ファイル: fileIO.py プロジェクト: Eragon666/ModerneDatabases
    def write(self, data):

        f = open(self.filename, "ba")
        offset = f.tell()
        f.write(add_integrity(encode(data)))
        f.close()
        return offset
コード例 #26
0
    def _commit(self, db):
        for n in self.bucket.values():
            n._commit(db)

        pos = db.tell()
        db.write(encode(self))
        return pos
コード例 #27
0
def senderResponse(s):
	global message
	if s == "0":
		makeSignal(message + encode(message), bitTime)
		return False
	else:
		return True
コード例 #28
0
ファイル: tmp.py プロジェクト: Eragon666/ModerneDatabases
def write_document(tofile, data):
    f = open(tofile, "ba")
    offset = f.tell()
    # print("offset: ", str(offset))
    f.write(add_integrity(encode(data)))
    f.close()
    return offset
コード例 #29
0
ファイル: tests.py プロジェクト: chenbk85/tinypy-panda
def t_render(ss, ex, exact=True):
    import tokenize, parse, encode

    if not istype(ss, 'list'): ss = [ss]
    n = 1
    for s in ss:
        fname = 'tmp' + str(n) + '.tpc'
        system_rm(fname)
        tokens = tokenize.tokenize(s)
        t = parse.parse(s, tokens)
        r = encode.encode(fname, s, t)
        f = save(fname, r)
        n += 1
    system_rm('tmp.txt')
    cmd = VM + fname + " > tmp.txt"
    system(cmd)
    res = load(TMP).strip()

    #print(ss,ex,res)
    if exact:
        if res != ex: showerror(cmd, ss, ex, res)
        assert (res == ex)
    else:
        if ex not in res: showerror(cmd, ss, ex, res)
        assert (ex in res)
コード例 #30
0
def main():
    ss = Preferences()
    script1sourcedir = ss.getwpath("script1sourcedir")            #("seeding\")
    files = [os.path.join(script1sourcedir,filename) for filename in next(os.walk(script1sourcedir))[2]]        #gives absolute paths + names

    currentfile = 0

    container = []    #set up an empty container for desired data to get put into for later
    for eachfile in files:

        metainfo = decoder.decode_from_file(eachfile)
        # #need to manually SHA1 hash the torrent file's info-dict to get the info-hash
        infodict = metainfo[b'info']
        info_hash = hashlib.sha1(encode.encode(infodict)).hexdigest().upper()

        internalname = infodict[b'name']
        torrentfilename = eachfile[eachfile.rfind("\\")+1:]
        locextension = torrentfilename.find(".torrent")           #location of extension (char position)
        locid = torrentfilename.rfind("-")+1                      #location of torrentID (char position)
        torrentid = torrentfilename[locid:locextension]           #grab torrentID 
      
        container.append([torrentfilename, internalname, info_hash, torrentid])
        currentfile += 1 
        print(currentfile, torrentfilename.encode('ascii', errors='ignore').decode())        #console output is ascii only, cannot print unicode - chars are omitted

    #WRITE FILE 1
    writelistfile = codecs.open(ss.getwpath("outpath1"),'wb',"utf-8") # write-out a text file with torrentID and Hash (on one line) ("1seeding_ID+Hash+Filename.txt")
    for eachline in container:
        writelistfile.write(eachline[3] + " / " + eachline[2] + " / " + eachline[0] + "\n")     #output torrentID / Hash / torrentfilename
    writelistfile.close()
コード例 #31
0
ファイル: tests.py プロジェクト: HarryR/tinypy-panda
def t_render(ss,ex,exact=True):
    import tokenize, parse, encode
        
    if not istype(ss,'list'): ss =[ss]
    n = 1
    for s in ss:
        fname = 'tmp'+str(n)+'.tpc'
        system_rm(fname)
        tokens = tokenize.tokenize(s)
        t = parse.parse(s,tokens)
        r = encode.encode(fname,s,t)
        f = save(fname,r)
        n += 1
    system_rm('tmp.txt')
    cmd = VM + fname + " > tmp.txt"
    system(cmd)
    res = load(TMP).strip()
    
    #print(ss,ex,res)
    if exact:
        if res != ex: showerror(cmd, ss, ex, res)
        assert(res == ex)
    else: 
        if ex not in res: showerror(cmd, ss, ex, res)
        assert(ex in res)
コード例 #32
0
def ckcode(e, r):
    c = encrypt(e + r)
    t = encrypt(e) + encode(encrypt(r))
    p = gen(t, c)
    u = encrypt(c)
    y = gen(p + gen(e, t) + u, t)
    return pricd(e, r, y)
コード例 #33
0
def encrypt():
    try:
        # creating new directory or checking if file exist or not
        os.path.exists("E:\\PROJ.RC\\python\\pyBot\\enc_files")
        #if file exist if will run except code
    except IOError:
        #if file exist call encode function
        encode()
    else:
        # otherwise (if there is no exception then it meas that file do not exist so create new file and call encode function)
        loop()
        print(" Directory does not exist")
        os.makedirs("E:\\PROJ.RC\\python\\pyBot\\enc_files")
        loop()
        print(" Created new file")
        encode()
        menu()
コード例 #34
0
ファイル: run.py プロジェクト: jerrylsu/latent-gan
 def run(self):
     print("Model LatentGAN running, encoding training set")
     encode(smiles_file=self.smiles_file,
            output_smiles_file_path=self.output_latent,
            encoder=self.encoder)
     print("Encoding finished finished. Creating model files")
     C = CreateModelRunner(input_data_path=self.output_latent,
                           output_model_folder=self.storage_path)
     C.run()
     print("Model Created. Training model")
     T = TrainModelRunner(input_data_path=self.output_latent,
                          output_model_folder=self.storage_path,
                          decode_mols_save_path=self.decoded_smiles,
                          n_epochs=self.n_epochs,
                          sample_after_training=self.sample_size)
     T.run()
     print("Model finished.")
コード例 #35
0
def ck(e, t, p, u, y, o):
    f = encrypt(e)
    g = encrypt(u + y)
    a = i = encrypt(e) + encode(encrypt(t)) + p
    b = gen_s(i, a)
    v = encrypt(f) + g
    j = gen_s(b + gen_s(e, a) + v, g)
    return prijm(e, t, p, u, y, o, j)
コード例 #36
0
    def test_gripper01_extracted_opgraph(self):
        file_dir = os.path.dirname(__file__) #<-- absolute dir the script is in
        original_sas = SAS3Extended.from_file(os.path.join(file_dir, 'test_cases/gripper01.sas'))
        encoded_sas = SAS3Extended.from_file(os.path.join(file_dir, 'test_cases/gripper01_extracted.sas'))

        candidates = extract_tau_operators_opgraph(original_sas)
        if len(candidates) > 0:
            original_sas = encode(original_sas,candidates)
        self.assertEqual(str(original_sas),str(encoded_sas))
コード例 #37
0
    def test_miconic_extracted_top_down(self):
        file_dir = os.path.dirname(__file__) #<-- absolute dir the script is in
        original_sas = SAS3Extended.from_file(os.path.join(file_dir, 'test_cases/miconic.sas'))
        expected_sas = SAS3Extended.from_file(os.path.join(file_dir, 'test_cases/miconic_extracted.sas'))

        candidates = extract_tau_operators_top(original_sas)
        if len(candidates) > 0:
            encoded_sas = encode(original_sas,candidates)
        self.assertEqual(str(encoded_sas),str(expected_sas))
コード例 #38
0
ファイル: idletalk.py プロジェクト: mehtank/rpnled
def send(data):
  sock = serial.Serial(target)
  time.sleep(0.1)
  e = encode(data)
  chunk = 40
  for s in (e[i:i+chunk] for i in range(0,len(e),chunk)):
    sock.write(s)
    time.sleep(0.1)
  sock.close()
コード例 #39
0
    def test_sokoban01_extracted_top_down(self):
        file_dir = os.path.dirname(__file__) #<-- absolute dir the script is in
        original_sas = SAS3Extended.from_file(os.path.join(file_dir, 'test_cases/sokoban01_essential.sas'))
        expected_sas = SAS3Extended.from_file(os.path.join(file_dir, 'test_cases/sokoban01_extracted.sas'))

        candidates = extract_tau_operators_top(original_sas)
        if len(candidates) > 0:
            encoded_sas = encode(original_sas,candidates)
        normalize(encoded_sas)
        self.assertMultiLineEqual(str(encoded_sas),str(expected_sas))
コード例 #40
0
def encodeAllFaces(imgs):
    faces = []
    for img in imgs:
        frame = cv2.imread(img)
        encoding = encode(frame)
        faces.extend(encoding)
        print("-- Processed {}: {} --".format(img, len(encoding)))

    # faces = [encode(cv2.imread(img)) for img in imgs]
    return np.array(faces)
コード例 #41
0
ファイル: tmp.py プロジェクト: Eragon666/ModerneDatabases
 def _commit(self):
     """
     Commit the changes. Calling the _commit() method of the root node, and
     writing its offset in the footer at the end of the file.
     """
     offset = self.root._commit()
     f = open(self.filename, "ba")
     f.write(add_integrity(encode({"root_offset":offset,
                                     "max_size":self.max_size})))
     f.close()
コード例 #42
0
    def write_piece(ty, node, out):
        lazy_parts = lazy.LazyMemberExtractor(types)
        node = lazy_parts.replace(ty, node)

        encode.encode(types, m, out, ty, node)

        # Encode the lazy parts in memory
        lazy_encoded = []
        for _, attr, part in lazy_parts.lazies:
            buf = io.BytesIO()
            lazy_encoded.append(buf)
            write_piece(attr.resolved_ty, part, buf)

        # Write the dictionary of lazy parts, then the lazy parts
        bits.write_varint(out, len(lazy_encoded))
        for encoded_part in lazy_encoded:
            bits.write_varint(out, encoded_part.tell())
        for encoded_part in lazy_encoded:
            out.write(encoded_part.getbuffer())
コード例 #43
0
ファイル: postprocessor.py プロジェクト: kop1/headphones
def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list):

    logger.info("Starting post-processing for: %s - %s" % (release["ArtistName"], release["AlbumTitle"]))
    # start enconding
    if headphones.ENCODE:
        downloaded_track_list = encode.encode(albumpath)

    if headphones.EMBED_ALBUM_ART or headphones.ADD_ALBUM_ART:
        album_art_path = albumart.getAlbumArt(albumid)
        artwork = urllib.urlopen(album_art_path).read()

    if headphones.EMBED_ALBUM_ART:
        embedAlbumArt(artwork, downloaded_track_list)

    if headphones.CLEANUP_FILES:
        cleanupFiles(albumpath)

    if headphones.ADD_ALBUM_ART:
        addAlbumArt(artwork, albumpath)

    if headphones.CORRECT_METADATA:
        correctMetadata(albumid, release, downloaded_track_list)

    if headphones.EMBED_LYRICS:
        embedLyrics(downloaded_track_list)

    if headphones.RENAME_FILES:
        renameFiles(albumpath, downloaded_track_list, release)

    if headphones.MOVE_FILES and headphones.DESTINATION_DIR:
        albumpath = moveFiles(albumpath, release, tracks)

    if headphones.MOVE_FILES and not headphones.DESTINATION_DIR:
        logger.error(
            'No DESTINATION_DIR has been set. Set "Destination Directory" to the parent directory you want to move the files to'
        )
        pass

    myDB = db.DBConnection()
    # There's gotta be a better way to update the have tracks - sqlite

    trackcount = myDB.select("SELECT HaveTracks from artists WHERE ArtistID=?", [release["ArtistID"]])

    if not trackcount[0][0]:
        cur_track_count = 0
    else:
        cur_track_count = trackcount[0][0]

    new_track_count = cur_track_count + len(downloaded_track_list)
    myDB.action("UPDATE artists SET HaveTracks=? WHERE ArtistID=?", [new_track_count, release["ArtistID"]])
    myDB.action('UPDATE albums SET status = "Downloaded" WHERE AlbumID=?', [albumid])
    myDB.action('UPDATE snatched SET status = "Processed" WHERE AlbumID=?', [albumid])
    updateHave(albumpath)

    logger.info("Post-processing for %s - %s complete" % (release["ArtistName"], release["AlbumTitle"]))
コード例 #44
0
ファイル: tests.py プロジェクト: HarryR/tinypy-panda
def t_unrender(s):
    import tokenize, parse, encode
        
    ok = False
    try:
        tokens = tokenize.tokenize(s)
        t = parse.parse(s,tokens)
        r = encode.encode('tmp.tpc',s,t)
    except:
        ok = True
    assert(ok == True)
コード例 #45
0
ファイル: tmp.py プロジェクト: Eragon666/ModerneDatabases
    def _get_data(self):
        """
        Returns the encoded data of the leaf node, containing its type, and the
        key/value pairs. These values will eventually be the offsets of the
        documents.
        """

        # print("Leaf committed: " + str(self) + " bucketsize: " +
        #     str(len(self.bucket)))
        data = {"type":"Leaf", "entries":self.bucket}
        # print("Leaf data: "+ str(data))
        return(add_integrity(encode(data)))
コード例 #46
0
ファイル: hostSocket.py プロジェクト: hacder/src
def socketSend(host,port,cmd,type=None):
	'''
	@host: 主机IP
	@port: 主机端口
	@cmd:  指令	
	'''

	if PortIsAlive(host, port,count=SOCKET['COUNT']) == False:
		return False
	
	data = d = ''
	
	try:
		sock = socket.socket()
		#settimeout后导致出现很多time out异常
		#参考文档:http://www.douban.com/note/174791641/
		#sock.settimeout(SOCKET['PROTIMEOUT'])
		
		sock.connect((host, port))
			
		if type:
			sock.send(encode(cmd + type))
		else:
			sock.send(encode(cmd))

		while True:
			d = sock.recv(SOCKET['BUFSIZE'])
			data = data + d
			time.sleep(0.003)
			if not d:
				sock.close()
				return data
			
				
	except Exception as e:
		
		save_log('ERROR','ip: '+host+' , port: '+str(port)+' , socketSend  error: '+str(e))
		sock.close()
		
		return False
コード例 #47
0
ファイル: hostSocket.py プロジェクト: hacder/src
def oldproSocket(host, port, cmd, type=None):
	'''
	@host: 主机IP
	@port: 主机端口
	@cmd:  指令
	'''

	# 指令执行后返回的结果
	data = ''

	# 开始通信
	try:
		sock = socket.socket()
		sock.settimeout(SOCKET['PROTIMEOUT'])
		sock.connect((host, port))
			
		if type:
			sock.send(encode(cmd + type))
		else:
			sock.send(encode(cmd))
		
		data = decode(sock.recv(SOCKET['BUFSIZE']))
		sock.close()
	
	except socket.timeout:
		sock.send(encode('Time out!'))
		save_log('WARNING','host:'+host+' Time out!')
		sock.close()
		return data
		sys.exit()
	
	except socket.error, args:
		(error_no, msg) = args
		error_log = 'Connect server faild:%s, error_no=%d ,error_host=%s' % (msg, error_no,host)
		save_log('ERROR',error_log)
		sock.close()
		return data
		sys.exit()
コード例 #48
0
ファイル: hostSocket.py プロジェクト: hacder/src
def FproSocket(host, port, cmd, type=None):
	'''
	@host: 主机IP
	@port: 主机端口
	@cmd:  指令
	'''

	 

	try:
		sock = socket.socket()
		sock.settimeout(SOCKET['PROTIMEOUT'])
		sock.connect((host, port))
                
		if type:
			sock.send(encode(cmd + type))
		else:
			sock.send(encode(cmd))
		sock.close()
	except Exception as e:
		log_error('FproSocket error:'+str(e))
		return 0
	return 1
コード例 #49
0
ファイル: worker.py プロジェクト: Yannf/video2commons
def main(self, url, ie_key, subtitles, filename, filedesc, convertkey, username, oauth):
    outputdir = generate_dir()
    s = stats()
    def statuscallback(text, percent):
        if text is not None: s.text = text
        if percent is not None: s.percent = percent
        print '%d: %s' % (s.percent, s.text)
        self.update_state(state='PROGRESS',
            meta={'text': s.text, 'percent': s.percent})

    def errorcallback(text):
        raise TaskError(text)

    statuscallback('Downloading...', -1)
    d = download.download(url, ie_key, 'bestvideo+bestaudio/best', subtitles, outputdir, statuscallback, errorcallback)
    if not d: errorcallback('Download failed!')
    file = d['target']
    if not file: errorcallback('Download failed!')
    subtitles = subtitles and d['subtitles']

    statuscallback('Converting...', -1)
    file = encode.encode(file, convertkey, statuscallback, errorcallback)
    if not file: errorcallback('Convert failed!')
    ext = file.split('.')[-1]

    statuscallback('Configuring Pywikibot...', -1)
    import pywikibot
    pywikibot.config.authenticate['commons.wikimedia.org'] = oauth
    pywikibot.config.usernames['commons']['commons'] = username
    pywikibot.Site('commons', 'commons', user=username).login()

    statuscallback('Uploading...', -1)
    fileurl = 'http://v2c.wmflabs.org/' + '/'.join(file.split('/')[3:])
    filename += '.' + ext
    filename, wikifileurl = upload.upload(file, filename, url, fileurl, filedesc, username, statuscallback, errorcallback)
    if not wikifileurl: errorcallback('Upload failed!')

    if subtitles:
        statuscallback('Uploading subtitles...', -1)
        try:
            subtitleuploader.subtitles(subtitles, filename, username, statuscallback, errorcallback)
        except Exception, e:
            statuscallback(type(e).__name__ + ": " + str(e), None)
            print e
            pass
コード例 #50
0
    def addClicked(self, **kwargs):

        fileName = ".data"
        # Writing encoding data to file
        if(kwargs['password'].get() != "" and kwargs['service'].get() != ""):
            data = None
            details = [kwargs['username'].get(),
                       encode.encode(kwargs['password'].get())]

            # Reading initally present data
            try:
                with open(fileName, "r") as outfile:
                    data = outfile.read()
            except IOError:
                # Create a file if it doesn't exits
                open(fileName, "a").close()

            # Loading new data
            if data:
                data = json.loads(data)
                data[kwargs['service'].get()] = details
            else:
                data = {}
                data[kwargs['service'].get()] = details

            # Writing back the data
            with open(".data", "w") as outfile:
                outfile.write(json.dumps(data, sort_keys=True, indent=4))

            # To delete contents of the Entry
            for widg in ('username', 'service', 'password'):
                kwargs[widg].delete(0, 'end')

            kwargs['info'].config(text="Added!!")

        # end of if
        else:

            kwargs['info'].config(text="Service or Password can't be empty!!")
コード例 #51
0
ファイル: postprocessor.py プロジェクト: SimonHova/headphones
def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list):

	logger.info('Starting post-processing for: %s - %s' % (release['ArtistName'], release['AlbumTitle']))
	#start enconding
	if headphones.ENCODE:
		downloaded_track_list=encode.encode(albumpath)
	
	album_art_path = albumart.getAlbumArt(albumid)
	
	if headphones.EMBED_ALBUM_ART or headphones.ADD_ALBUM_ART:
		artwork = urllib.urlopen(album_art_path).read()
		if len(artwork) < 100:
			artwork = False
			logger.info("No suitable album art found. Not adding album art")
	
	if headphones.EMBED_ALBUM_ART and artwork:
		embedAlbumArt(artwork, downloaded_track_list)
	
	if headphones.CLEANUP_FILES:
		cleanupFiles(albumpath)
		
	if headphones.ADD_ALBUM_ART and artwork:
		addAlbumArt(artwork, albumpath)
		
	if headphones.CORRECT_METADATA:
		correctMetadata(albumid, release, downloaded_track_list)
		
	if headphones.EMBED_LYRICS:
		embedLyrics(downloaded_track_list)
		
	if headphones.RENAME_FILES:
		renameFiles(albumpath, downloaded_track_list, release)
	
	if headphones.MOVE_FILES and headphones.DESTINATION_DIR:
		albumpath = moveFiles(albumpath, release, tracks)
	
	if headphones.MOVE_FILES and not headphones.DESTINATION_DIR:
		logger.error('No DESTINATION_DIR has been set. Set "Destination Directory" to the parent directory you want to move the files to')
		pass
		
	myDB = db.DBConnection()
	# There's gotta be a better way to update the have tracks - sqlite
	
	trackcount = myDB.select('SELECT HaveTracks from artists WHERE ArtistID=?', [release['ArtistID']])
	
	if not trackcount[0][0]:
		cur_track_count = 0
	else:
		cur_track_count = trackcount[0][0]
		
	new_track_count = cur_track_count + len(downloaded_track_list)
	myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [new_track_count, release['ArtistID']])
	myDB.action('UPDATE albums SET status = "Downloaded" WHERE AlbumID=?', [albumid])
	myDB.action('UPDATE snatched SET status = "Processed" WHERE AlbumID=?', [albumid])
	updateHave(albumpath)
	
	logger.info('Post-processing for %s - %s complete' % (release['ArtistName'], release['AlbumTitle']))
	
	if headphones.PROWL_ONSNATCH:
		pushmessage = release['ArtistName'] + ' - ' + release['AlbumTitle']
		logger.info(u"Prowl request")
		prowl = notifiers.PROWL()
		prowl.notify(pushmessage,"Download and Postprocessing completed")
		
	if headphones.XBMC_ENABLED:
		xbmc = notifiers.XBMC()
		if headphones.XBMC_UPDATE:
			xbmc.update()
		if headphones.XBMC_NOTIFY:
			xbmc.notify(release['ArtistName'], release['AlbumTitle'], album_art_path)
コード例 #52
0
ファイル: py2bc.py プロジェクト: ashmew2/kolibriosSVN
def _compile(s,fname):
    tokens = tokenize.tokenize(s)
    t = parse.parse(s,tokens)
    r = encode.encode(fname,s,t)
    return r
コード例 #53
0
ファイル: main.py プロジェクト: Wysaat/qrcode
    sys.exit(0)
rawdata = sys.argv[1:-1]
ecl = sys.argv[-1]
data = ' '.join(rawdata)

# step 1. data analysis
if set(data).issubset(set(numeric)):
    mode = NUMERIC
elif set(data).issubset(set(alphanumeric)):
    mode = ALPHANUMERIC
elif set(data).issubset(set(eightbitbyte)):
    mode = EIGHTBITBYTE
print "mode determined... is", mode

version = utils.getversion(data, mode)
print "version dtermined... is", version

print "generating codewords now... please wait"
codewords = encode.encode(data, version, mode, ecl)

print "generating finalmessage now... please wait"
finalmessage = errorcode.genfinalmessage(codewords, version, ecl)

print "generating matrix now... please wait"
fmatrix = matrix.getmatrix(version, finalmessage, ecl)

print "draw the qrcode now... please wait"
draw.drawmatrix(fmatrix)
print "..."
print "done!"
コード例 #54
0
ファイル: talk.py プロジェクト: mehtank/rpnled
def send(data):
  sock.send(encode(data))
コード例 #55
0
ファイル: preprocess.py プロジェクト: jiamaozheng/rail
def go(nucleotides_per_input=8000000, gzip_output=True, gzip_level=3,
        to_stdout=False, push='.', mover=filemover.FileMover(),
        verbose=False, scratch=None, bin_qualities=True, short_qnames=False,
        skip_bad_records=False, workspace_dir=None,
        fastq_dump_exe='fastq-dump', ignore_missing_sra_samples=False):
    """ Runs Rail-RNA-preprocess

        Input (read from stdin)
        ----------------------------
        Tab-separated fields:
        ---If URL is local:
        1. #!splitload
        2. \x1d-separated list of 0-based indexes of reads at which to start
            each new file
        3. \x1d-separated list of numbers of reads to include in gzipped files
        4. \x1d-separated list of manifest lines whose tabs are replaced by
            \x1es

        ---Otherwise:
        manifest line

        A manifest line has the following format

        (for single-end reads)
        <URL>(tab)<Optional MD5>(tab)<Sample label>

        (for paired-end reads)
        <URL 1>(tab)<Optional MD5 1>(tab)<URL 2>(tab)<Optional MD5 2>(tab)
        <Sample label>

        Hadoop output (written to stdout)
        ----------------------------
        None.

        Other output (written to directory specified by command-line parameter
            --push)
        ____________________________
        Files containing input data in one of the following formats:

        Format 1 (single-end, 3-column):
          1. Nucleotide sequence or its reversed complement, whichever is first
            in alphabetical order
          2. 1 if sequence was reverse-complemented else 0
          3. Name
          4. Quality sequence or its reverse, whichever corresponds to field 1

        Format 2 (paired, 2 lines, 3 columns each)
        (so this is the same as single-end)
          1. Nucleotide sequence for mate 1 or its reversed complement,
            whichever is first in alphabetical order
          2. 1 if sequence was reverse-complemented else 0
          3. Name for mate 1
          4. Quality sequence for mate 1 or its reverse, whichever corresponds
            to field 1
            
            (new line)

          1. Nucleotide sequence for mate 2 or its reversed complement,
            whichever is first in alphabetical order
          2. 1 if sequence was reverse complemented else 0
          3. Name for mate 2
          4. Quality sequence for mate 2 or its reverse, whichever corresponds
            to field 1

        Quality sequences are strings of Is for FASTA input.

        nucleotides_per_input: maximum number of nucleotides to put in a given
            input file
        gzip_output: True iff preprocessed input should be gzipped
        gzip_level: level of gzip compression to use
        push: where to send output
        verbose: True iff extra debugging statements should be printed to
            stderr
        scratch: scratch directory for storing temporary files or None if 
            securely created temporary directory
        bin_qualities: True iff quality string should be binned according to
            rules in _mismatch_penalties_to_quality_scores
            and round_quality_string() defined in go()
        short_qnames: True iff original qname should be killed and a new qname
            should be written in a short base64-encoded format
        skip_bad_records: True iff bad records should be skipped; otherwise,
            raises exception if bad record is encountered
        workspace_dir: where to use fastq-dump -- needed for working with
            dbGaP data. None if temporary dir should be used.
        fastq_dump_exe: path to fastq-dump executable
        ignore_missing_sra_samples: does not return error if fastq-dump doesn't
            find a sample

        No return value
    """
    if bin_qualities:
        import math
        def round_quality_string(qual):
            """ Bins phred+33 quality string to improve compression.

                Uses 5-bin scheme that does not affect Bowtie 2 alignments

                qual: quality string

                Return value: "binned" quality string.
            """
            return ''.join(
                [str(int(
                    _MN + math.floor((_MX - _MN) * min(
                                                    ord(qual_char) - 33.0, 40.0
                                                ) / 40.0)
                        )) for qual_char in qual]).translate(
                                _mismatch_penalties_to_quality_scores
                            )
    else:
        def round_quality_string(qual):
            """ Leaves quality string unbinned and untouched.

                qual: quality string

                Return value: qual
            """
            return qual
    global _input_line_count, _output_line_count
    skip_stubs = False
    temp_dir = make_temp_dir(scratch)
    print >>sys.stderr, 'Created local destination directory "%s".' % temp_dir
    register_cleanup(tempdel.remove_temporary_directories, [temp_dir])
    input_line_count, output_line_count = 0, 0
    if not to_stdout:
        push_url = Url(push)
        if push_url.is_local:
            destination = push
        elif push_url.is_s3 or push_url.is_hdfs or push_url.is_nfs:
            destination = temp_dir
        else:
            raise RuntimeError('Push destination must be '
                               'on S3, HDFS, NFS, or local.')
    fastq_cues = set(['@'])
    fasta_cues = set(['>', ';'])
    source_dict = {}
    onward = False
    for line in sys.stdin:
        _input_line_count += 1
        if not line.strip(): continue
        # Kill offset from start of manifest file
        try:
            tokens = line.strip().split('\t')[1:]
            if tokens[0][0] == '#' and tokens[0] != '#!splitload':
                # Comment line
                continue
        except IndexError:
            # Be robust to bad lines
            continue
        token_count = len(tokens)
        qual_getter = None
        if tokens[0] == '#!splitload':
            '''Line specifies precisely how records from files should be
            placed.'''
            assert not to_stdout, ('Split manifest line inconsistent with '
                                   'writing to stdout.')
            qual_getter = phred_converter(phred_format=tokens[-1])
            indexes = tokens[1].split('\x1d')
            read_counts = tokens[2].split('\x1d')
            manifest_lines = [token.split('\x1e')
                                for token in tokens[3].split('\x1d')]
            assert len(indexes) == len(read_counts) == len(manifest_lines)
            for i, manifest_line in enumerate(manifest_lines):
                manifest_line_field_count = len(manifest_line)
                if manifest_line_field_count == 3:
                    source_dict[(Url(manifest_line[0]),)] = (
                            manifest_line[-1],
                            int(indexes[i]),
                            int(read_counts[i])
                        )
                else:
                    assert manifest_line_field_count == 5
                    source_dict[(Url(manifest_line[0]),
                                 Url(manifest_line[2]))] = (
                                                        manifest_line[-1],
                                                        int(indexes[i]),
                                                        int(read_counts[i])
                                                    )
        elif token_count == 3:
            # SRA or single-end reads
            source_dict[(Url(tokens[0]),)] = (tokens[-1],)
        elif token_count == 5:
            # Paired-end reads
            source_dict[(Url(tokens[0]), Url(tokens[2]))] = (tokens[-1],)
        else:
            # Not a valid line, but continue for robustness
            continue
    file_number = 0
    for source_urls in source_dict:
        sample_label = source_dict[source_urls][0]
        downloaded = set()
        sources = []
        records_printed = 0
        if len(source_dict[source_urls]) == 3:
            skip_count = source_dict[source_urls][1]
            if len(source_urls) == 2:
                records_to_consume = source_dict[source_urls][2]
                if skip_count % 2:
                    skip_count -= 1
                    records_to_consume += 1
                if records_to_consume % 2:
                    records_to_consume -= 1
                # Index reads according to order in input to shorten read names
                read_index = skip_count / 2 # Index reads in pairs
            else:
                records_to_consume = source_dict[source_urls][2]
                read_index = skip_count
        else:
            skip_count = 0
            records_to_consume = None # Consume all records
            read_index = 0
        assert (records_to_consume >= 0 or records_to_consume is None), (
                'Negative value %d of records to consume encountered.'
            ) % records_to_consume
        if records_to_consume == 0: continue
        skipped = False
        for source_url in source_urls:
            if not source_url.is_local:
                # Download
                print >>sys.stderr, 'Retrieving URL "%s"...' \
                    % source_url.to_url()
                if source_url.is_dbgap:
                    download_dir = workspace_dir
                elif source_url.is_sra:
                    download_dir = temp_dir
                if source_url.is_sra:
                    sra_accession = source_url.to_url()
                    fastq_dump_command = (
                            'set -exo pipefail; cd {download_dir}; '
                            '{fastq_dump_exe} -I -X 10000 --split-files '
                            '{sra_accession}'
                        ).format(download_dir=download_dir,
                                    fastq_dump_exe=fastq_dump_exe,
                                    sra_accession=sra_accession)
                    try:
                        subprocess.check_call(
                            fastq_dump_command, shell=True, 
                            executable='/bin/bash',
                            stdout=sys.stderr
                        )
                    except subprocess.CalledProcessError as e:
                        if e.returncode == 3 and ignore_missing_sra_samples:
                            onward = True
                            break
                        else:
                            raise RuntimeError(
                                ('Error "%s" encountered executing '
                                 'command "%s".') % (e.output,
                                                        fastq_dump_command))
                    import glob
                    sra_fastq_files = sorted(
                                        glob.glob(os.path.join(download_dir,
                                            '%s[_.]*' % sra_accession))
                                        ) # ensure 1 before 2 if paired-end
                    # Schedule for deletion
                    def silent_remove(filename):
                        try:
                            os.remove(filename)
                        except OSError as e:
                            pass
                    for sra_fastq_file in sra_fastq_files:
                        register_cleanup(silent_remove, sra_fastq_file)
                    sra_file_count = len(sra_fastq_files)
                    check_for_paired = False
                    if sra_file_count == 1:
                        sra_paired_end = False
                        print >>sys.stderr, 'Detected single-end SRA sample.'
                    elif sra_file_count in [2, 3]:
                        print >>sys.stderr, ('2 or 3 FASTQ files detected. '
                                             'Checking for barcodes...')
                        check_for_paired = True
                    else:
                        raise RuntimeError(
                                ('Unexpected number of files "%d" output '
                                 'by fastq-dump command "%s".')
                                    % (sra_file_count, fastq_dump_command)
                            )
                    if check_for_paired:
                        # Get max/min read lengths from FASTQ
                        with open(
                                    sra_fastq_files[sra_file_count - 2]
                                ) as fastq_stream:
                            max_len, min_len = (
                                    max_min_read_lengths_from_fastq_stream(
                                        fastq_stream
                                    )
                                )
                            print >>sys.stderr, (
                                    'Max/min read length found in candidate '
                                    'barcode FASTQ was {}/{}.'
                                ).format(max_len, min_len)
                            if max_len <= _max_stubby_read_length:
                                print >>sys.stderr, (
                                        'Assumed barcode FASTQ.'
                                    )
                                skip_stubs = True
                                if sra_file_count == 2:
                                    sra_paired_end = False
                                else:
                                    sra_paired_end = True
                            else:
                                if sra_file_count == 2:
                                    sra_paired_end = True
                                else:
                                    raise RuntimeError(
                                        '3 FASTQs detected, but one of them '
                                        'was not recognized as containing '
                                        'barcodes.'
                                    )
                    # Guess quality from first 10k lines
                    with xopen(None, sra_fastq_files[0]) as source_stream:
                        qual_getter = phred_converter(
                                            fastq_stream=source_stream
                                        )
                    for sra_fastq_file in sra_fastq_files:
                        os.remove(sra_fastq_file)
                    sources.append(os.devnull)
                    fastq_dump_command = (
                            'set -exo pipefail; cd {download_dir}; '
                            '{fastq_dump_exe} --split-spot -I --stdout '
                            '{sra_accession}'
                        ).format(download_dir=download_dir,
                                    fastq_dump_exe=fastq_dump_exe,
                                    sra_accession=sra_accession)
                    if skip_stubs:
                        fastq_dump_command += (
                                ' | awk \'BEGIN {{OFS = "\\n"}} '
                                '{{header = $0; '
                                'getline seq; getline qheader; getline qseq; '
                                'if (length(seq) > {min_len}) {{print header, '
                                'seq, qheader, qseq}}}}\''
                            ).format(min_len=_max_stubby_read_length)
                    print >>sys.stderr, fastq_dump_command
                    sra_process = subprocess.Popen(fastq_dump_command,
                                                    shell=True,
                                                    executable='/bin/bash',
                                                    stdout=subprocess.PIPE,
                                                    bufsize=-1)
                else:
                    mover.get(source_url, temp_dir)
                    downloaded = list(
                            set(os.listdir(temp_dir)).difference(downloaded)
                        )
                    sources.append(os.path.join(temp_dir, list(downloaded)[0]))
            else:
                sources.append(source_url.to_url())
        if onward: continue
        '''Use os.devnull so single- and paired-end data can be handled in one
        loop.'''
        if len(sources) == 1:
            sources.append(os.devnull)
        if qual_getter is None:
            # Figure out Phred format
            with xopen(None, sources[0]) as source_stream:
                qual_getter = phred_converter(fastq_stream=source_stream)
        with xopen(None, sources[0]) as source_stream_1, xopen(
                None, sources[1]
            ) as source_stream_2:
            source_streams = [source_stream_1, source_stream_2]
            reorganize = all([source == os.devnull for source in sources])
            if reorganize:
                # SRA data is live
                if sra_paired_end:
                    source_streams = [sra_process.stdout, sra_process.stdout]
                else:
                    source_streams = [sra_process.stdout, open(os.devnull)]
            break_outer_loop = False
            while True:
                if not to_stdout:
                    '''Name files using Hadoop task environment property
                    mapred.task.partition.'''
                    if gzip_output:
                        try:
                            output_file = os.path.join(
                                    destination, 
                                    '.'.join([
                                        os.environ['mapred_task_partition'],
                                        str(file_number), 'gz'
                                    ])
                                )
                        except KeyError:
                            '''Hadoop 2.x: mapreduce.task.partition; see 
                            http://hadoop.apache.org/docs/r2.0.3-alpha/
                            hadoop-project-dist/hadoop-common/
                            DeprecatedProperties.html.'''
                            output_file = os.path.join(
                                    destination, 
                                    '.'.join([
                                        os.environ['mapreduce_task_partition'],
                                        str(file_number), 'gz'
                                    ])
                                )
                        open_args = [output_file, 'a', gzip_level]
                    else:
                        try:
                            output_file = os.path.join(
                                    destination, 
                                    '.'.join([
                                        os.environ['mapred_task_partition'],
                                        str(file_number)
                                    ])
                                )
                        except KeyError:
                            output_file = os.path.join(
                                    destination, 
                                    '.'.join([
                                        os.environ['mapreduce_task_partition'],
                                        str(k), str(file_number)
                                    ])
                                )
                        open_args = [output_file, 'a']
                    try:
                        os.makedirs(os.path.dirname(output_file))
                    except OSError:
                        pass
                else:
                    open_args = []
                '''Use xopen to handle compressed streams and normal streams
                generally.'''
                with xopen(gzip_output if not to_stdout else '-', *open_args) \
                    as output_stream:
                    perform_push = False
                    line_numbers = [0, 0]
                    read_next_line = True
                    nucs_read = 0
                    pairs_read = 0
                    while True:
                        if read_next_line:
                            # Read next line only if FASTA mode didn't already
                            lines = []
                            for source_stream in source_streams:
                                lines.append(source_stream.readline())
                        read_next_line = True
                        if not lines[0]:
                            break_outer_loop = True
                            break
                        line_numbers = [i + 1 for i in line_numbers]
                        lines = [line.strip() for line in lines]
                        bad_record_skip = False
                        if lines[0][0] in fastq_cues:
                            if records_to_consume and not skipped:
                                '''Skip lines as necessary; for paired-end
                                reads skip the largest even number of records 
                                less than records_to_consume.'''
                                if len(source_urls) == 1:
                                    # single-end
                                    line_skip_count = max(
                                            skip_count * 4 - 1, 0
                                        )
                                else:
                                    # paired-end
                                    line_skip_count = max(
                                            ((skip_count / 2) * 4 - 1), 0
                                        )
                                    for _ in xrange(line_skip_count):
                                        next(source_stream_2)
                                for _ in xrange(line_skip_count):
                                    next(source_stream_1)
                                if skip_count:
                                    lines = []
                                    for source_stream in source_streams:
                                        lines.append(source_stream.readline())
                                    if not lines[0]:
                                        break_outer_loop = True
                                        break
                                    lines = [line.strip() for line in lines]
                                skipped = True
                            seqs = [source_stream.readline().strip()
                                        for source_stream in source_streams]
                            line_numbers = [i + 1 for i in line_numbers]
                            plus_lines = [source_stream.readline().strip()
                                            for source_stream
                                            in source_streams]
                            line_numbers = [i + 1 for i in line_numbers]
                            quals = [source_stream.readline().strip()
                                        for source_stream in source_streams]
                            if reorganize and sra_paired_end:
                                # Fix order!
                                lines, seqs, plus_lines, quals = (
                                        [lines[0], plus_lines[0]],
                                        [lines[1], plus_lines[1]],
                                        [seqs[0], quals[0]],
                                        [seqs[1], quals[1]]
                                    )
                            try:
                                assert plus_lines[0][0] == '+', (
                                        'Malformed read "%s" at line %d of '
                                        'file "%s".'
                                    ) % (lines[0], line_numbers[0], sources[0])
                                if plus_lines[1]:
                                    assert plus_lines[1][0] == '+', (
                                            'Malformed read "%s" at line %d '
                                            'of file "%s".'
                                        ) % (
                                        lines[1], line_numbers[1], sources[1]
                                    )
                                try:
                                    # Kill spaces in name
                                    original_qnames = \
                                        [line[1:].replace(' ', '_')
                                            for line in lines]
                                except IndexError:
                                    raise RuntimeError(
                                            'Error finding QNAME at ' 
                                            'line %d of either %s or %s' % (
                                                        sources[0],
                                                        sources[1]
                                                    )
                                        )
                            except (AssertionError,
                                    IndexError, RuntimeError) as e:
                                if skip_bad_records:
                                    print >>sys.stderr, ('Error "%s" '
                                            'encountered; skipping bad record.'
                                        ) % e.message
                                    for source_stream in source_streams:
                                        source_stream.readline()
                                    line_numbers = [
                                            i + 1 for i in line_numbers
                                        ]
                                    bad_record_skip = True
                                else:
                                    raise
                            else:
                                try:
                                    quals = [
                                            qual_getter(qual) for qual in quals
                                        ]
                                except Exception as e:
                                    if skip_bad_records:
                                        print >>sys.stderr, (
                                                'Error "%s" encountered '
                                                'trying to convert quality '
                                                'string to Sanger format; '
                                                'skipping bad record.'
                                            ) % e.message
                                        bad_record_skip = True
                                    else:
                                        raise
                                line_numbers = [i + 1 for i in line_numbers]
                                try: 
                                    for i in xrange(2):
                                        assert len(seqs[i]) == len(quals[i]), (
                                            'Length of read sequence does not '
                                            'match length of quality string '
                                            'at line %d of file "%s".'
                                        ) % (line_numbers[i], sources[i])
                                except (AssertionError, IndexError) as e:
                                    if skip_bad_records:
                                        print >>sys.stderr, (
                                                'Error "%s" encountered; '
                                                'skipping bad record.'
                                            ) % e.message
                                        bad_record_skip = True
                                    else:
                                        raise
                        elif lines[0][0] in fasta_cues:
                            seqs = [[], []]
                            next_lines = []
                            for p, source_stream in enumerate(source_streams):
                                while True:
                                    next_line \
                                        = source_stream.readline().strip()
                                    try:
                                        if next_line[0] in fasta_cues:
                                            break
                                        else:
                                            try:
                                                seqs[p].append(next_line)
                                            except IndexError:
                                                raise
                                    except IndexError:
                                        break
                                next_lines.append(next_line)
                            seqs = [''.join(seq) for seq in seqs]
                            line_numbers = [i + 1 for i in line_numbers]
                            try:
                                try:
                                    # Kill spaces in name
                                    original_qnames = \
                                        [line[1:].replace(' ', '_')
                                            for line in lines]
                                except IndexError:
                                    raise RuntimeError(
                                            'Error finding QNAME at ' 
                                            'line %d of either %s or %s' % (
                                                        sources[0],
                                                        sources[1]
                                                    )
                                        )
                            except (AssertionError,
                                    IndexError, RuntimeError) as e:
                                if skip_bad_records:
                                    print >>sys.stderr, ('Error "%s" '
                                            'encountered; skipping bad record.'
                                        ) % e.message
                                    for source_stream in source_streams:
                                        source_stream.readline()
                                    line_numbers = [
                                            i + 1 for i in line_numbers
                                        ]
                                    bad_record_skip = True
                                else:
                                    raise
                            else:
                                try:
                                    quals = [
                                        'h'*len(seq) for seq in seqs
                                        ]
                                except Exception as e:
                                    if skip_bad_records:
                                        print >>sys.stderr, (
                                                'Error "%s" encountered '
                                                'trying to convert quality '
                                                'string to Sanger format; '
                                                'skipping bad record.'
                                            ) % e.message
                                        bad_record_skip = True
                                    else:
                                        raise
                                line_numbers = [i + 1 for i in line_numbers]
                            lines = next_lines
                            read_next_line = False
                        if bad_record_skip:
                            seqs = []
                            # Fake record-printing to get to records_to_consume
                            if source_streams[-1].name == os.devnull:
                                records_printed += 1
                            else:
                                records_printed += 2
                        elif len(original_qnames) == 2 and original_qnames[1]:
                            # Paired-end write
                            if original_qnames[0] == original_qnames[1]:
                                # Add paired-end identifiers
                                original_qnames[0] += '/1'
                                original_qnames[1] += '/2'
                            assert seqs[1]
                            assert quals[1]
                            seqs = [seq.upper() for seq in seqs]
                            reversed_complement_seqs = [
                                    seqs[0][::-1].translate(
                                        _reversed_complement_translation_table
                                    ),
                                    seqs[1][::-1].translate(
                                        _reversed_complement_translation_table
                                    )
                                ]
                            if seqs[0] < reversed_complement_seqs[0]:
                                left_seq = seqs[0]
                                left_qual = quals[0]
                                left_reversed = '0'
                            else:
                                left_seq = reversed_complement_seqs[0]
                                left_qual = quals[0][::-1]
                                left_reversed = '1'
                            if seqs[1] < reversed_complement_seqs[1]:
                                right_seq = seqs[1]
                                right_qual = quals[1]
                                right_reversed = '0'
                            else:
                                right_seq = reversed_complement_seqs[1]
                                right_qual = quals[1][::-1]
                                right_reversed = '1'
                            if short_qnames:
                                left_qname_to_write = encode(read_index) + '/1'
                                right_qname_to_write = encode(
                                                            read_index
                                                        ) + '/2'
                            else:
                                left_qname_to_write = original_qnames[0]
                                right_qname_to_write = original_qnames[1]
                            print >>output_stream, '\t'.join(
                                        [
                                            left_seq,
                                            left_reversed,
                                            qname_from_read(
                                                    left_qname_to_write,
                                                    seqs[0] + quals[0], 
                                                    sample_label,
                                                    mate=seqs[1]
                                                ),
                                            '\n'.join([
                                                round_quality_string(
                                                    left_qual
                                                ), right_seq
                                            ]),
                                            right_reversed,
                                            qname_from_read(
                                                    right_qname_to_write,
                                                    seqs[1] + quals[1], 
                                                    sample_label,
                                                    mate=seqs[0]
                                                ),
                                            round_quality_string(right_qual)
                                        ]
                                    )
                            records_printed += 2
                            _output_line_count += 1
                        else:
                            seqs[0] = seqs[0].upper()
                            reversed_complement_seqs = [
                                    seqs[0][::-1].translate(
                                        _reversed_complement_translation_table
                                    )
                                ]
                            # Single-end write
                            if seqs[0] < reversed_complement_seqs[0]:
                                seq = seqs[0]
                                qual = quals[0]
                                is_reversed = '0'
                            else:
                                seq = reversed_complement_seqs[0]
                                qual = quals[0][::-1]
                                is_reversed = '1'
                            if short_qnames:
                                qname_to_write = encode(read_index)
                            else:
                                qname_to_write = original_qnames[0]
                            print >>output_stream, '\t'.join(
                                        [
                                            seq,
                                            is_reversed,
                                            qname_from_read(
                                                qname_to_write,
                                                seqs[0] + quals[0], 
                                                sample_label
                                            ),
                                            round_quality_string(qual)
                                        ]
                                    )
                            records_printed += 1
                            _output_line_count += 1
                        read_index += 1
                        for seq in seqs:
                            nucs_read += len(seq)
                        if records_printed == records_to_consume:
                            break_outer_loop = True
                            perform_push = True
                            break
                        if not to_stdout and not records_to_consume and \
                            nucs_read > nucleotides_per_input:
                            file_number += 1
                            break
                if verbose:
                    print >>sys.stderr, (
                            'Exited with statement; line numbers are %s' 
                            % line_numbers
                        )
                if (not to_stdout) and (push_url.is_nfs or
                    push_url.is_s3 or push_url.is_hdfs) \
                    and ((not records_to_consume) or
                         (records_to_consume and perform_push)):
                    print >>sys.stderr, 'Pushing "%s" to "%s" ...' % (
                                                            output_file,
                                                            push_url.to_url()
                                                        )
                    print >>sys.stderr, 'reporter:status:alive'
                    mover.put(output_file, push_url.plus(os.path.basename(
                                                                output_file
                                                            )))
                    try:
                        os.remove(output_file)
                    except OSError:
                        pass
                if break_outer_loop: break
            if verbose:
                print >>sys.stderr, 'Exiting source streams...'
        if verbose:
            print >>sys.stderr, 'Exited source streams.'
        # Clear temporary directory
        for input_file in os.listdir(temp_dir):
            try:
                os.remove(os.path.join(temp_dir, input_file))
            except OSError:
                pass
        if 'sra_process' in locals():
            sra_process.stdout.close()
            sra_return_code = sra_process.wait()
            if sra_return_code > 0:
                raise RuntimeError(('fastq-dump terminated with exit '
                                    'code %d. Command run was "%s".')
                                        % (sra_return_code,
                                            fastq_dump_command))
            del sra_process
コード例 #56
0
cluster_size = 8

# column 1: number of pieces
# column 2: size of a bitfield message
# column 3: size of compressed bitfield message (worst case)
# column 4: size of compressed bitfield message (realistic)

bitfield = ['0'] * num_pieces
for i in xrange(num_pieces + 1):

	scattered_pieces = gen_scattered_pieces(i, 0, num_pieces)
	for pos in scattered_pieces:
		bitfield[pos] = '1'

	# divide by 3 because each byte is HEX encoded with spaces in between
	scatter_size = len(encode.encode(''.join(bitfield))) / 3
	realistic_size = len(encode.encode_pieces( \
		gen_realistic_pieces(i, num_pieces, cluster_size))) / 3
	out.write('%d\t%d\t%d\t%d\n' % (i, \
		5 + (num_pieces + 7)/8, \
		4 + 2 + scatter_size, \
		4 + 2 + realistic_size))

#	print ''.join(bitfield)

out.close()

f = open('bitfield-sizes.gnuplot', 'w+')
f.write('''
set term png small size 640,480
set output "bitfield-sizes.png"
コード例 #57
0
ファイル: worker.py プロジェクト: JeanFred/video2commons
def main(
    self, url, ie_key, subtitles, filename, filedesc,
    downloadkey, convertkey, username, oauth
):
    """Main worker code."""
    # Get a lock to prevent double-running with same task ID
    lockkey = 'tasklock:' + self.request.id
    if redisconnection.exists(lockkey):
        raise TaskError("Task has already been run")
    else:
        redisconnection.setex(lockkey, 'T', 7 * 24 * 3600)

    # Generate temporary directory for task
    for i in range(10):  # 10 tries
        id = os.urandom(8).encode('hex')
        outputdir = '/srv/v2c/output/' + id
        if not os.path.isdir(outputdir):
            os.mkdir(outputdir)
            break
    else:
        raise TaskError("Too many retries to generate a task id")

    s = Stats()

    def statuscallback(text, percent):
        if text is not None:
            s.text = text
        if percent is not None:
            s.percent = percent
        print '%d: %s' % (s.percent, s.text)

        self.update_state(
            state='PROGRESS',
            meta={'text': s.text, 'percent': s.percent}
        )

    def errorcallback(text):
        raise TaskError(text)

    try:
        statuscallback('Downloading...', -1)
        d = download.download(
            url, ie_key, downloadkey, subtitles,
            outputdir, statuscallback, errorcallback
        )
        if not d:
            errorcallback('Download failed!')
        file = d['target']
        if not file:
            errorcallback('Download failed!')
        subtitles = subtitles and d['subtitles']

        statuscallback('Converting...', -1)
        file = encode.encode(file, convertkey, statuscallback, errorcallback)
        if not file:
            errorcallback('Convert failed!')
        ext = file.split('.')[-1]

        statuscallback('Configuring Pywikibot...', -1)
        pywikibot.config.authenticate['commons.wikimedia.org'] = \
            (consumer_key, consumer_secret) + oauth
        pywikibot.config.usernames['commons']['commons'] = username
        pywikibot.Site('commons', 'commons', user=username).login()

        statuscallback('Uploading...', -1)
        filename += '.' + ext
        filename, wikifileurl = upload.upload(
            file, filename, url, http_host,
            filedesc, username, statuscallback, errorcallback
        )
        if not wikifileurl:
            errorcallback('Upload failed!')

        if subtitles:
            statuscallback('Uploading subtitles...', -1)
            try:
                subtitleuploader.subtitles(
                    subtitles, filename, username,
                    statuscallback, errorcallback
                )
            except Exception, e:
                statuscallback(type(e).__name__ + ": " + str(e), None)
                print e
                pass

    except pywikibot.Error:  # T124922 workaround
        exc_info = sys.exc_info()
        raise TaskError(
            (
                u'pywikibot.Error: %s: %s' % (
                    exc_info[0].__name__, exc_info[1]
                )
            ).encode('utf-8')), None, exc_info[2]
    else:
        statuscallback('Done!', 100)
        return filename, wikifileurl
    finally:
        statuscallback('Cleaning up...', -1)
        pywikibot.config.authenticate.clear()
        pywikibot.config.usernames['commons'].clear()
        pywikibot._sites.clear()

        shutil.rmtree(outputdir)