Exemplo n.º 1
0
def write_csv(filename, header, data):
    file_result = open(filename, "wb")
    c = UnicodeWriter(file_result, delimiter=';', quoting=csv.QUOTE_ALL)
    c.writerow(header)
    for d in data:
        c.writerow(d)
    file_result.close()
Exemplo n.º 2
0
def write_file(filename, header, data, fail=False, model="auto", launchfile="import_auto.sh", worker=1, batch_size=10, init=False):
    def get_model():
        if model == "auto":
            return filename.split('/')[-1][:-4]
        else:
            return model

    file_result = open(filename, "wb")
    c = UnicodeWriter(file_result, delimiter=';', quoting=csv.QUOTE_ALL)
    c.writerow(header)
    for d in data:
        c.writerow(d)
    file_result.close()

    mode = init and 'w' or 'a'
    with open(launchfile, mode) as myfile:
        myfile.write("python odoo_import_thread.py -c conf/connection.conf --file=%s --model=%s --worker=%s --size=%s \n" % (filename, get_model(), worker, batch_size))
        if fail:
            myfile.write("python odoo_import_thread.py -c conf/connection.conf --fail --file=%s --model=%s --worker=%s --size=%s \n" % (filename, get_model(), worker, batch_size))
Exemplo n.º 3
0
def write_csv(filename, header, data):
    file_result = open(filename, "wb")
    c = UnicodeWriter(file_result, delimiter=';', quoting=csv.QUOTE_ALL)
    c.writerow(header)
    for d in data:
        c.writerow(d)
    file_result.close()
Exemplo n.º 4
0
connection = conf_lib.get_server_connection(config_file)
object_registry = connection.get_model(model)
ir_model_registry = connection.get_model('ir.model.data')

header = reader.next()
try:
    id_index = header.index('id')
except ValueError as ve:
    print "No External Id (id) column defined, please add one"
    raise ve

i = 1
file_result = open(fail_file, "wb")

c = UnicodeWriter(file_result)
c.writerow(header)
file_result.flush()
for line in reader:
    st = time()
    lines = [line]
    xml_ids = [line[id_index]]
    success = False
    j = 1
    while j < batch_size:
        j += 1
        i += 1
        line = reader.next()
        lines.append(line)
        xml_ids.append(line[id_index])
def retrieve_category_metadata(input_csv_filename, output_csv_similarity_filename, out_csv_genre_filename, out_csv_popularity_filename):

	# retrieve data
	categories_array = read_csv(input_csv_filename)

	output_file_similarity = open(output_csv_similarity_filename, 'wb')
	#writer_similarity = csv.writer(output_file_similarity)
	writer_similarity = UnicodeWriter(output_file_similarity)
	writer_similarity.writerow(['categoryID', 'categoryName', 'spotifyID', 'spotifyName', 'spotifyArtistID', 'spotifyArtistName'])

	output_file_genre = open(out_csv_genre_filename, 'wb')
	#writer_genre = csv.writer(output_file_genre)
	writer_genre = UnicodeWriter(output_file_genre)
	writer_genre.writerow(['categoryID', 'categoryName', 'spotifyID', 'spotifyName', 'genre', 'weight'])

	output_file_popularity = open(out_csv_popularity_filename, 'wb')
	#writer_popularity = csv.writer(output_file_popularity)
	writer_popularity = UnicodeWriter(output_file_popularity)
	writer_popularity.writerow(['categoryID', 'categoryName', 'spotifyID', 'spotifyName', 'spotifyPopularity'])

	counter = 1
	matched = 0
	no_match = 0
	for category_dict in categories_array:
		categoryID = category_dict['categoryID']
		categoryName = (category_dict['categoryName']).lower()

		# sleep for a bit to respect API calls throtlling
		time.sleep(RATE_LIMIT_SLEEP_TIME)

		try:
			res = search_artist_http(categoryName.encode('utf-8'))

		except:
			res = {}

		if not res:
			# this means the dict was empty
			print "#%d.(catName: %s; catId:%s) retrieved nothing." %(counter, categoryName, categoryID)
			no_match = no_match +1

		else:
			spotify_name = (res['spotify_name']).decode('utf-8')
			spotify_id = str(res['spotify_id']) 
			spotify_popularity = str(res['spotify_popularity'])

			spotify_genres = res['spotify_genres']
			artists = res['similar_artists']
			writer_popularity.writerow([categoryID, categoryName, spotify_id, spotify_name, spotify_popularity])

			for genre_dict in spotify_genres:
				genre_name = genre_dict['spotify_genre']
				genre_weight = str(genre_dict['weight'])

				writer_genre.writerow([categoryID, categoryName, spotify_id, spotify_name, genre_name, genre_weight])

			for artist in artists:
				artist_name = artist['spotify_name']
				artist_id = str(artist['spotify_id'])

				writer_similarity.writerow([categoryID, categoryName, spotify_id, spotify_name, artist_id, artist_name])


			matched = matched + 1

			counter = counter + 1
			#print "........"
			#if (counter > 10):
			#	break
			
	print "..........................................." 
	print "Done"
	print "Total Processed = %d" %(counter)
	print "Spotify Matches found = %d" %(matched)
	print "No Matches = %d" %(no_match)
	print "..........................................." 
Exemplo n.º 6
0
header = header[:header_len]
        
try:
    id_index = header.index('id')
except ValueError as ve:
    print "No External Id (id) column defined, please add one"
    raise ve
    
i = 1
#while i < 6870:
#    reader.next()
#    i+= 1;
    
file_result = open(fail_file, "wb")

c = UnicodeWriter(file_result, delimiter=';')
c.writerow(header)
file_result.flush()
thread_list = []
st = time()
for line in reader:
    lines = [line[:header_len]]
        
    j = 1
    while j < batch_size and line:
        j += 1
        i += 1
        try:
            line = reader.next()[:header_len]
            lines.append(line)
            
connection = conf_lib.get_server_connection(config_file)
object_registry = connection.get_model(model)
ir_model_registry = connection.get_model('ir.model.data')

header = reader.next()
try:
    id_index = header.index('id')
except ValueError as ve:
    print "No External Id (id) column defined, please add one"
    raise ve
    
i = 1
file_result = open(fail_file, "wb")

c = UnicodeWriter(file_result)
c.writerow(header)
file_result.flush()
for line in reader:
    st = time()
    lines = [line]
    xml_ids = [line[id_index]]
    success = False
    j = 1
    while j < batch_size:
        j += 1
        i += 1
        line = reader.next()
        lines.append(line)
        xml_ids.append(line[id_index])
        
Exemplo n.º 8
0
header = header[:header_len]

try:
    id_index = header.index('id')
except ValueError as ve:
    print "No External Id (id) column defined, please add one"
    raise ve

i = 1
#while i < 6870:
#    reader.next()
#    i+= 1;

file_result = open(fail_file, "wb")

c = UnicodeWriter(file_result, delimiter=';')
c.writerow(header)
file_result.flush()
thread_list = []
st = time()
for line in reader:
    lines = [line[:header_len]]

    j = 1
    while j < batch_size and line:
        j += 1
        i += 1
        try:
            line = reader.next()[:header_len]
            lines.append(line)