def write_csv(filename, header, data):
file_result = open(filename, "wb")
c = UnicodeWriter(file_result, delimiter=';', quoting=csv.QUOTE_ALL)
c.writerow(header)
for d in data:
c.writerow(d)
file_result.close()
def write_file(filename, header, data, fail=False, model="auto", launchfile="import_auto.sh", worker=1, batch_size=10, init=False):
def get_model():
if model == "auto":
return filename.split('/')[-1][:-4]
else:
return model
file_result = open(filename, "wb")
c = UnicodeWriter(file_result, delimiter=';', quoting=csv.QUOTE_ALL)
c.writerow(header)
for d in data:
c.writerow(d)
file_result.close()
mode = init and 'w' or 'a'
with open(launchfile, mode) as myfile:
myfile.write("python odoo_import_thread.py -c conf/connection.conf --file=%s --model=%s --worker=%s --size=%s \n" % (filename, get_model(), worker, batch_size))
if fail:
myfile.write("python odoo_import_thread.py -c conf/connection.conf --fail --file=%s --model=%s --worker=%s --size=%s \n" % (filename, get_model(), worker, batch_size))
def write_csv(filename, header, data):
file_result = open(filename, "wb")
c = UnicodeWriter(file_result, delimiter=';', quoting=csv.QUOTE_ALL)
c.writerow(header)
for d in data:
c.writerow(d)
file_result.close()
connection = conf_lib.get_server_connection(config_file)
object_registry = connection.get_model(model)
ir_model_registry = connection.get_model('ir.model.data')
header = reader.next()
try:
id_index = header.index('id')
except ValueError as ve:
print "No External Id (id) column defined, please add one"
raise ve
i = 1
file_result = open(fail_file, "wb")
c = UnicodeWriter(file_result)
c.writerow(header)
file_result.flush()
for line in reader:
st = time()
lines = [line]
xml_ids = [line[id_index]]
success = False
j = 1
while j < batch_size:
j += 1
i += 1
line = reader.next()
lines.append(line)
xml_ids.append(line[id_index])
def retrieve_category_metadata(input_csv_filename, output_csv_similarity_filename, out_csv_genre_filename, out_csv_popularity_filename):
# retrieve data
categories_array = read_csv(input_csv_filename)
output_file_similarity = open(output_csv_similarity_filename, 'wb')
#writer_similarity = csv.writer(output_file_similarity)
writer_similarity = UnicodeWriter(output_file_similarity)
writer_similarity.writerow(['categoryID', 'categoryName', 'spotifyID', 'spotifyName', 'spotifyArtistID', 'spotifyArtistName'])
output_file_genre = open(out_csv_genre_filename, 'wb')
#writer_genre = csv.writer(output_file_genre)
writer_genre = UnicodeWriter(output_file_genre)
writer_genre.writerow(['categoryID', 'categoryName', 'spotifyID', 'spotifyName', 'genre', 'weight'])
output_file_popularity = open(out_csv_popularity_filename, 'wb')
#writer_popularity = csv.writer(output_file_popularity)
writer_popularity = UnicodeWriter(output_file_popularity)
writer_popularity.writerow(['categoryID', 'categoryName', 'spotifyID', 'spotifyName', 'spotifyPopularity'])
counter = 1
matched = 0
no_match = 0
for category_dict in categories_array:
categoryID = category_dict['categoryID']
categoryName = (category_dict['categoryName']).lower()
# sleep for a bit to respect API calls throtlling
time.sleep(RATE_LIMIT_SLEEP_TIME)
try:
res = search_artist_http(categoryName.encode('utf-8'))
except:
res = {}
if not res:
# this means the dict was empty
print "#%d.(catName: %s; catId:%s) retrieved nothing." %(counter, categoryName, categoryID)
no_match = no_match +1
else:
spotify_name = (res['spotify_name']).decode('utf-8')
spotify_id = str(res['spotify_id'])
spotify_popularity = str(res['spotify_popularity'])
spotify_genres = res['spotify_genres']
artists = res['similar_artists']
writer_popularity.writerow([categoryID, categoryName, spotify_id, spotify_name, spotify_popularity])
for genre_dict in spotify_genres:
genre_name = genre_dict['spotify_genre']
genre_weight = str(genre_dict['weight'])
writer_genre.writerow([categoryID, categoryName, spotify_id, spotify_name, genre_name, genre_weight])
for artist in artists:
artist_name = artist['spotify_name']
artist_id = str(artist['spotify_id'])
writer_similarity.writerow([categoryID, categoryName, spotify_id, spotify_name, artist_id, artist_name])
matched = matched + 1
counter = counter + 1
#print "........"
#if (counter > 10):
# break
print "..........................................."
print "Done"
print "Total Processed = %d" %(counter)
print "Spotify Matches found = %d" %(matched)
print "No Matches = %d" %(no_match)
print "..........................................."
header = header[:header_len]
try:
id_index = header.index('id')
except ValueError as ve:
print "No External Id (id) column defined, please add one"
raise ve
i = 1
#while i < 6870:
# reader.next()
# i+= 1;
file_result = open(fail_file, "wb")
c = UnicodeWriter(file_result, delimiter=';')
c.writerow(header)
file_result.flush()
thread_list = []
st = time()
for line in reader:
lines = [line[:header_len]]
j = 1
while j < batch_size and line:
j += 1
i += 1
try:
line = reader.next()[:header_len]
lines.append(line)
connection = conf_lib.get_server_connection(config_file)
object_registry = connection.get_model(model)
ir_model_registry = connection.get_model('ir.model.data')
header = reader.next()
try:
id_index = header.index('id')
except ValueError as ve:
print "No External Id (id) column defined, please add one"
raise ve
i = 1
file_result = open(fail_file, "wb")
c = UnicodeWriter(file_result)
c.writerow(header)
file_result.flush()
for line in reader:
st = time()
lines = [line]
xml_ids = [line[id_index]]
success = False
j = 1
while j < batch_size:
j += 1
i += 1
line = reader.next()
lines.append(line)
xml_ids.append(line[id_index])
header = header[:header_len]
try:
id_index = header.index('id')
except ValueError as ve:
print "No External Id (id) column defined, please add one"
raise ve
i = 1
#while i < 6870:
# reader.next()
# i+= 1;
file_result = open(fail_file, "wb")
c = UnicodeWriter(file_result, delimiter=';')
c.writerow(header)
file_result.flush()
thread_list = []
st = time()
for line in reader:
lines = [line[:header_len]]
j = 1
while j < batch_size and line:
j += 1
i += 1
try:
line = reader.next()[:header_len]
lines.append(line)
